Testing query with query broker C++

compose.yaml

@@ -0,0 +1,132 @@
+
+services:
+  redis:
+    image: redis:7.2.3-alpine
+    container_name: test-das-redis
+    restart: always
+    command: redis-server --save 20 1 --loglevel warning
+    ports:
+      - 6379:6379
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
+      interval: 10s
+      timeout: 10s
+      retries: 5
+      start_period: 40s
+
+  mongodb:
+    image: mongo:latest
+    container_name: test-das-mongodb
+    restart: always
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: root
+      MONGO_INITDB_ROOT_PASSWORD: root
+    ports:
+      - 27017:27017
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "echo",
+          'db.runCommand("ping").ok',
+          "|",
+          "mongo",
+          "das-mongo:27017/test",
+          "--quiet",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 5
+      start_period: 40s
+
+
+  metta-parser:
+#    image: trueagi/das:latest-metta-parser
+    image: trueagi/das:0.5.3-metta-parser
+    container_name: test-das-metta-parser
+    command: db_loader /tmp/animals.metta
+    volumes:
+      - ./examples/data:/tmp
+    environment:
+      DAS_MONGODB_NAME: das
+      DAS_MONGODB_HOSTNAME: localhost
+      DAS_MONGODB_PORT: 27017
+      DAS_REDIS_HOSTNAME: localhost
+      DAS_REDIS_PORT: 6379
+      DAS_MONGODB_USERNAME: root
+      DAS_MONGODB_PASSWORD: root
+      DAS_DATABASE_USERNAME: root
+      DAS_DATABASE_PASSWORD: root
+#    env_file:
+#      - .env
+    depends_on:
+      mongodb:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    restart: on-failure
+
+
+  das-node:
+    # How to build:
+    # Clone the das-node repository
+    # Run:
+    # ./scripts/docker_image_build.sh
+#    image: das-node-builder
+    build: ./docker_das_node
+#    build: https://github.com/singnet/das-attention-broker.git#master:docker
+#    build:
+#      context: https://raw.githubusercontent.com/singnet/das-attention-broker/refs/heads/master/docker/Dockerfile
+    container_name: test-das-node
+#    working_dir: /opt/das-attention-broker
+#    command: ./query_broker 35700
+#    command: ./bin 35700
+#    command: ./query_broker 35700
+    command: sleep infinity
+    network_mode: "host"
+    ports:
+      - 35700:35700
+    depends_on:
+      - mongodb
+      - redis
+      - metta-parser
+    environment:
+      DAS_MONGODB_NAME: das
+      DAS_MONGODB_HOSTNAME: test-das-mongodb
+      DAS_MONGODB_PORT: 27017
+      DAS_REDIS_HOSTNAME: test-das-redis
+      DAS_REDIS_PORT: 6379
+      DAS_MONGODB_USERNAME: root
+      DAS_MONGODB_PASSWORD: root
+#      DAS_DATABASE_USERNAME: root
+#      DAS_DATABASE_PASSWORD: root
+#    command: sleep infinity
+    #    command: bash -c "cd src && ../scripts/bazel_build.sh && cd .. && pip install . && python3 -i examples/docker_server.py"
+
+  fass:
+#    image: trueagi/openfaas:query-engine-1.14.12
+    image: trueagi/openfaas:query-engine-1.14.13
+#    image: trueagi/openfaas:query-engine-latest
+    container_name: test-das-faas
+#    privileged: true
+    depends_on:
+      - mongodb
+      - redis
+      - metta-parser
+      - das-node
+    ports:
+      - 8080:8080
+    environment:
+      DAS_MONGODB_NAME: test-das
+      DAS_MONGODB_HOSTNAME: test-das-mongodb
+      DAS_MONGODB_PORT: 27017
+      DAS_REDIS_HOSTNAME: test-das-redis
+      DAS_REDIS_PORT: 6379
+      DAS_MONGODB_USERNAME: root
+      DAS_MONGODB_PASSWORD: root
+      DAS_DATABASE_USERNAME: root
+      DAS_DATABASE_PASSWORD: root
+
+
+
+

docker_das_node/Dockerfile

@@ -0,0 +1,187 @@
+# FROM das-node-builder
+# ARG BASE_DIR="/opt"
+# ARG ATTENTION_BROKER_DIR="${BASE_DIR}/das-attention-broker"
+# ARG THIRDPARTY="${ATTENTION_BROKER_DIR}/assets"
+#
+# WORKDIR /opt/hyperon_das_node/src
+# RUN ../scripts/bazel_build.sh
+#
+#
+# WORKDIR /opt
+# RUN git clone https://github.com/singnet/das-attention-broker.git
+# WORKDIR /opt/das-attention-broker/
+# RUN ls -la
+# RUN cd ${THIRDPARTY}\
+#     && tar xzvf 3rd-party.tgz\
+#     && rm -f 3rd-party.tgz\
+#     && mkdir -p ${ATTENTION_BROKER_DIR}/src/3rd-party\
+#     && ln -s /opt/3rd-party ${ATTENTION_BROKER_DIR}/src/3rd-party
+#
+# ENV CPLUS_INCLUDE_PATH="/opt/3rd-party/mbedcrypto/include/"
+#
+#
+# WORKDIR /opt/das-attention-broker/src
+# RUN ls -l
+# RUN pwd
+# # RUN ../scripts/bazel_build.sh
+# RUN /opt/bazel/bazelisk build --jobs 6 --noenable_bzlmod :query_broker
+# # RUN mv bazel-bin/query_broker ../bin
+# WORKDIR /opt/das-attention-broker
+#
+# CMD .bin/query_broker
+#
+# # RUN pip install --no-cache-dir hyperon_das_node
+# #
+# # WORKDIR /opt/hyperon_das_node
+# #
+# # COPY *.py .
+# #
+# #
+# # # Server image for docker example, exposes server port
+# # FROM base AS server
+# #
+# # EXPOSE 35700
+# #
+# # ENTRYPOINT ["python", "-i", "das_node_server.py"]
+
+
+
+FROM ubuntu:22.04
+
+ARG BASE_DIR="/opt"
+ARG TMP_DIR="/tmp"
+
+ARG ATTENTION_BROKER_DIR="${BASE_DIR}/das-attention-broker"
+ARG DATA_DIR="${BASE_DIR}/data"
+ARG GRPC_DIR="${BASE_DIR}/grpc"
+ARG PROTO_DIR="${BASE_DIR}/proto"
+ARG BAZEL_DIR="${BASE_DIR}/bazel"
+ARG THIRDPARTY="${BASE_DIR}/3rd-party"
+
+# RUN mkdir -p ${ATTENTION_BROKER_DIR}
+RUN mkdir -p ${DATA_DIR}
+RUN mkdir -p ${GRPC_DIR}
+RUN mkdir -p ${BAZEL_DIR}
+RUN mkdir -p ${PROTO_DIR}
+RUN mkdir -p ${THIRDPARTY}
+VOLUME ${ATTENTION_BROKER_DIR}
+
+RUN apt-get update -y
+RUN apt-get install -y git
+
+RUN cd ${GRPC_DIR} &&\
+    git clone https://github.com/grpc/grpc &&\
+    cd grpc &&\
+    git submodule update --init
+
+RUN apt-get install -y build-essential autoconf libtool pkg-config curl gcc protobuf-compiler libmbedtls14 libmbedtls-dev libevent-dev libssl-dev
+# RUN apt-get install -y autoconf
+# RUN apt-get install -y libtool
+# RUN apt-get install -y pkg-config
+# RUN apt-get install -y curl
+# RUN apt-get install -y gcc
+# RUN apt-get install -y protobuf-compiler
+# #RUN apt-get install -y libmbedcrypto7
+# RUN apt-get install -y libmbedtls14
+# RUN apt-get install -y libmbedtls-dev
+# RUN apt-get install -y libevent-dev
+# RUN apt-get install -y libssl-dev
+
+################################################################################
+# To be removed when AtomDB is properly integrated
+# Redis client
+RUN apt-get install -y cmake libevent-dev libssl-dev pkg-config cmake-data git
+
+WORKDIR ${BASE_DIR}
+RUN git clone https://github.com/singnet/das-attention-broker.git
+WORKDIR ${ATTENTION_BROKER_DIR}
+
+RUN sed -i '24i\AtomDBSingleton::init();' src/main/query_engine_main.cc
+RUN cat src/main/query_engine_main.cc
+
+RUN cp assets/hiredis-cluster.tgz /tmp
+
+RUN cd /tmp &&\
+    tar xzf hiredis-cluster.tgz &&\
+    cd hiredis-cluster &&\
+    mkdir build &&\
+    cd build &&\
+    cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_SSL=ON ..&&\
+    make &&\
+    make install &&\
+    echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf &&\
+    ldconfig
+# MongoDB client
+WORKDIR ${ATTENTION_BROKER_DIR}
+
+# RUN cp assets/mongo-cxx-driver-r3.11.0.tar.gz /tmp &&\
+#     cd /tmp && \
+#     tar xzvf mongo-cxx-driver-r3.11.0.tar.gz &&\
+#     cd /tmp/mongo-cxx-driver-r3.11.0/build/ &&\
+#     cmake .. -DCMAKE_BUILD_TYPE=Release -DMONGOCXX_OVERRIDE_DEFAULT_INSTALL_PREFIX=OFF &&\
+#     cmake --build .
+RUN cp assets/mongo-cxx-driver-r3.11.0.tar.gz /tmp
+WORKDIR ${TMP_DIR}
+RUN tar xzvf mongo-cxx-driver-r3.11.0.tar.gz
+WORKDIR ${TMP_DIR}/mongo-cxx-driver-r3.11.0/build
+RUN cmake .. -DCMAKE_BUILD_TYPE=Release -DMONGOCXX_OVERRIDE_DEFAULT_INSTALL_PREFIX=OFF &&\
+    cmake --build . &&\
+    cmake --build . --target install
+RUN pwd &&\
+    # ls -a   &&\
+    # ls /usr/local/include/mongocxx/v_noabi &&\
+    # ls -a .. &&\
+    # mv install/include/* /usr/local/include &&\
+    ln -s /usr/local/include/bsoncxx/v_noabi/bsoncxx/* /usr/local/include/bsoncxx &&\
+    ln -s /usr/local/include/bsoncxx/v_noabi/bsoncxx/third_party/mnmlstc/core/ /usr/local/include/core &&\
+    ln -s /usr/local/include/mongocxx/v_noabi/mongocxx/* /usr/local/include/mongocxx/ &&\
+    # mv install/lib/* /usr/local/lib &&\
+    ldconfig
+# RUN cmake --build . --target install &&\
+#     mv install/include/* /usr/local/include &&\
+#     ln -s /usr/local/include/bsoncxx/v_noabi/bsoncxx/* /usr/local/include/bsoncxx &&\
+#     ln -s /usr/local/include/bsoncxx/v_noabi/bsoncxx/third_party/mnmlstc/core/ /usr/local/include/core &&\
+#     ln -s /usr/local/include/mongocxx/v_noabi/mongocxx/* /usr/local/include/mongocxx/ &&\
+#     mv install/lib/* /usr/local/lib &&\
+#     ldconfig
+################################################################################
+WORKDIR ${ATTENTION_BROKER_DIR}
+RUN mkdir bin
+
+RUN cp assets/3rd-party.tgz ${THIRDPARTY}
+RUN cd ${THIRDPARTY} &&\
+    tar xzvf 3rd-party.tgz &&\
+    rm -f 3rd-party.tgz &&\
+    mkdir -p ${ATTENTION_BROKER_DIR}/src/3rd-party &&\
+    ln -s ${THIRDPARTY} ${ATTENTION_BROKER_DIR}/src/3rd-party &&\
+    mv bazelisk ${BAZEL_DIR}
+
+ENV CPLUS_INCLUDE_PATH="/opt/3rd-party/mbedcrypto/include/"
+
+ENV CC=/usr/bin/gcc
+RUN ln -s ${BAZEL_DIR}/bazelisk /usr/bin/bazel
+# RUN cd ${GRPC_DIR}/grpc &&\
+#     ${BAZEL_DIR}/bazelisk build :all
+
+# RUN cd ${GRPC_DIR}/grpc &&\
+#     ${BAZEL_DIR}/bazelisk build --jobs 6 --noenable_bzlmod :query_broker
+# RUN ${BAZEL_DIR}/bazelisk build --jobs 6 --noenable_bzlmod :query_broker
+WORKDIR ${ATTENTION_BROKER_DIR}
+RUN rm -rf  bin
+RUN mkdir bin
+WORKDIR ${ATTENTION_BROKER_DIR}/src
+# RUN ${BAZEL_DIR}/bazelisk build --jobs 6 --noenable_bzlmod :query_broker
+# RUN mv bazel-bin/query_broker ../query_broker
+RUN ../scripts/bazel_build.sh
+# RUN cp bazel-bin/query_broker ../
+# build --jobs 6 --noenable_bzlmod :query_broker
+ADD https://raw.githubusercontent.com/singnet/das-query-engine/master/proto/attention_broker.proto ${PROTO_DIR}
+ADD https://raw.githubusercontent.com/singnet/das-query-engine/master/proto/common.proto ${PROTO_DIR}
+ADD https://raw.githubusercontent.com/singnet/das-query-engine/master/proto/echo.proto ${PROTO_DIR}
+EXPOSE 35700
+WORKDIR /opt/das-attention-broker
+
+RUN ls -la
+# RUN ls -la bin/
+# RUN ls  /opt/bazel
+# RUN ls  /opt

hyperon_das/das_node/das_node.py

@@ -0,0 +1,74 @@
+from typing import List, Optional, Dict
+import uuid
+from hyperon_das.das_node.star_node import StarNode
+from hyperon_das.das_node.remote_iterator import RemoteIterator
+from hyperon_das_node import Message, MessageFactory
+from hyperon_das.das_node.query_element import QueryElement
+
+
+
+class DASNode(StarNode):
+    PATTERN_MATCHING_QUERY = "pattern_matching_query"
+
+    def __init__(self, node_id: str = None, server_id: str = None):
+        super().__init__(node_id, server_id)
+        self.initialize()
+
+    def __del__(self):
+        # Destructor (although Python handles memory management automatically)
+        pass
+
+    def pattern_matcher_query(self, tokens: list, context: str = ""):
+        if self.is_server:
+            raise ValueError("pattern_matcher_query() is not available in DASNode server.")
+
+        # TODO: Update this when requestor is set in basic Message
+        query_id = self.next_query_id()
+        # print(query_id, tokens)
+        args = [query_id, context] + tokens
+        print(self.server_id)
+        print(args)
+        self.send(DASNode.PATTERN_MATCHING_QUERY, args, self.server_id)
+
+
+        print('send')
+        return RemoteIterator(query_id)
+
+    def next_query_id(self) -> str:
+        port = self.next_query_port
+        limit = 0
+        if self.is_server:
+            limit = (self.first_query_port + self.last_query_port) // 2 - 1
+            if self.next_query_port > limit:
+                self.next_query_port = self.first_query_port
+        else:
+            limit = self.last_query_port
+            if self.next_query_port > limit:
+                self.next_query_port = (self.first_query_port + self.last_query_port) // 2
+
+        query_id = f"{self.local_host}:{port}"
+        self.next_query_port += 1
+        return query_id
+
+    def message_factory(self, command: str, args: list) -> Message:
+        message = super().message_factory(command, args)
+        if message:
+            return message
+        if command == DASNode.PATTERN_MATCHING_QUERY:
+            return PatternMatchingQuery(command, args)
+        return None
+
+    def initialize(self):
+        self.first_query_port = 60000
+        self.last_query_port = 61999
+        self.local_host = self.node_id().split(":")[0]  # Extracting the host part of node_id
+        if self.is_server:
+            self.next_query_port = self.first_query_port
+        else:
+            self.next_query_port = (self.first_query_port + self.last_query_port) // 2
+
+    # Private attributes (Python doesn't have access specifiers, but this is by convention)
+    local_host: str
+    next_query_port: int
+    first_query_port: int
+    last_query_port: int