CRT constraints and AMM benchmarks

.dockerignore

@@ -4,8 +4,6 @@ target/
 .vscode/
 query-node/**/dist
 query-node/lib
-tests/
-!tests/network-tests/proposal-parameters.json
 devops/
 !devops/eslint-config
-!devops/prettier-config
+!devops/prettier-config

.env

@@ -1,10 +1,11 @@
 COMPOSE_PROJECT_NAME=joystream
 PROJECT_NAME=query_node
 
-# We will use a single postgres service with multiple databases
-# The env variables below are by default used by all services and should be
-# overriden in local env files
-# DB config
+# We use a single postgres service for both the query node indexer and processor.
+# The default `DB_*` environment variables point to query node processor's database.
+# The DB_NAME env is overriden with INDEXER_DB_NAME in the indexer's service docker-compose config.
+# When running other services, those values should be overriden by their local environment configs.
+# Query node DB config:
 INDEXER_DB_NAME=query_node_indexer
 DB_NAME=query_node_processor
 DB_USER=postgres
@@ -90,6 +91,44 @@ HCAPTCHA_SECRET=
 BALANCE_CREDIT=300000000000
 BALANCE_LOCKED=300000000000
 
+# ====================================== ORION ======================================
+# Orion db config
+# Overrides `DB_*` environment variables for orion services in docker-compose.yml
+# We use a single postgres service for both:
+# - Subsquid archive's database (squid-archive, squid-archive-gateway) and
+# - Orion's database (orion_processor, orion_graphql-server).
+# ORION_DB_NAME is the name of the Orion's database, while SQUID_ARCHIVE_DB_NAME
+# is the name of the Subsquid archive database.
+SQUID_ARCHIVE_DB_NAME=squid_archive
+ORION_DB_NAME=orion
+ORION_DB_USER=postgres
+ORION_DB_PASS=postgres
+ORION_DB_PORT=5433
+
+# Orion / Subsquid archive service ports
+ORION_API_PORT=4350
+PROCESSOR_PROMETHEUS_PORT=3337
+SQUID_ARCHIVE_GATEWAY_PORT=8000
+
+# Other Orion configs
+SUPPORT_NO_CATEGORY_VIDEOS=true
+SUPPORT_NEW_CATEGORIES=true
+KILL_SWITCH_ON=false
+VIDEO_VIEW_PER_IP_TIME_LIMIT=30
+VIDEO_RELEVANCE_VIEWS_TICK=50
+RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5]"
+OPERATOR_SECRET=this-is-not-so-secret-change-it
+MAX_CACHED_ENTITIES=1000
+APP_PRIVATE_KEY=this-is-not-so-secret-change-it
+TRUSTED_REVERSE_PROXIES=1
+# =====================================================================================
+
+## Telemetry
+# yes/no
+TELEMETRY_ENABLED=no
+# TELEMETRY_ENDPOINT=http://apm-server:8200
+TELEMETRY_ENDPOINT=http://collector:4318
+
 # joystream/node docker image tag
 # We do not provide a default value - scripts that startup a joystream-node service
 # Should be explicit about what version to use.

.github/workflows/create-release.yml

@@ -65,12 +65,12 @@ jobs:
 
           docker run -d --entrypoint tail --name temp-container-joystream-node $IMAGE-amd64 -f /dev/null
 
-          RESULT=$(docker exec temp-container-joystream-node b2sum -l 256 runtime.compact.wasm | awk '{print $1}')
+          RESULT=$(docker exec temp-container-joystream-node b2sum -l 256 runtime.compact.compressed.wasm | awk '{print $1}')
           VERSION_AND_COMMIT=$(docker exec temp-container-joystream-node /joystream/node --version | awk '{print $2}' | cut -d- -f -2)
           echo "::set-output name=blob_hash::${RESULT}"
           echo "::set-output name=version_and_commit::${VERSION_AND_COMMIT}"
 
-          docker cp temp-container-joystream-node:/joystream/runtime.compact.wasm ./joystream_runtime_${{ github.event.inputs.tag }}.wasm
+          docker cp temp-container-joystream-node:/joystream/runtime.compact.compressed.wasm ./joystream_runtime_${{ github.event.inputs.tag }}.wasm
           docker cp temp-container-joystream-node:/joystream/node ./joystream-node
           tar -czvf joystream-node-$VERSION_AND_COMMIT-x86_64-linux-gnu.tar.gz joystream-node
 

.github/workflows/deploy-node-network.yml

@@ -86,6 +86,10 @@ jobs:
             EC2InstanceType=${{ steps.network_config.outputs.instanceType }},
             VolumeSize=${{ steps.network_config.outputs.volumeSize }}
 
+      - name: Wait for docker build server to be ready
+        run: |
+          sleep 30
+
       - name: Prepare inventory for Ansible
         run: |
           VAL1="${{ steps.deploy_stack.outputs.Val1PublicIp }}"

.github/workflows/deploy-playground.yml

@@ -86,6 +86,10 @@ jobs:
           # Make sure ami image is available in the region specified in configure aws creds step
           parameter-overrides: 'KeyName=joystream-github-action-key-new,EC2InstanceType=t2.xlarge,EC2AMI=ami-06b4d9ba1f23a8da4'
 
+      - name: Wait for docker build server to be ready
+        run: |
+          sleep 30
+
       - name: Run playbook
         uses: dawidd6/action-ansible-playbook@v2
         with:

.github/workflows/joystream-apps-docker.yml

@@ -1,30 +1,103 @@
-name: Build joystream/apps and publish to Docker Hub
+name: Publish Colossus/Argus/QueryNode Docker images
 
 on:
   workflow_dispatch:
-    inputs:
-      tag_suffix:
-        description: 'Tag suffix'
-        required: true
+  pull_request:
+
 jobs:
-  docker:
+  build_and_publish:
+    name: Build and Publish
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Extract branch name
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+
+      - name: Extract Package Versions
+        id: extract_versions
         shell: bash
-        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: extract_branch
+        run: |
+          echo "colossus_version=$(cat storage-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
+          echo "argus_version=$(cat distributor-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
+          echo "qn_version=$(cat query-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
+
+      - name: Make some space
+        shell: bash
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
+        if: github.event_name == 'workflow_dispatch'
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
-      - name: Build and push
-        id: docker_build
+
+      - name: Build joystream-node
+        run: RUNTIME_PROFILE=TESTING ./build-node-docker.sh
+
+      # docker/build-push-action doc:
+      # Be careful because any file mutation in the steps that precede the
+      # build step will be ignored, including processing of the .dockerignore file
+      # since the context is based on the Git reference. However, you can use
+      # the Path context using the context input alongside the actions/checkout action
+      # to remove this restriction.
+      - name: Build storage-node
         uses: docker/build-push-action@v3
         with:
-          file: apps.Dockerfile
-          push: true
-          tags: joystream/apps:${{ steps.extract_branch.outputs.branch }}-${{ github.event.inputs.tag_suffix }}
+          # Do not use local dir context to ensure we can build from a commit directly
+          # context: .
+          file: colossus.Dockerfile
+          push: false
+          load: true
+          tags: joystream/storage-node:latest
+      - name: Build distributor-node
+        uses: docker/build-push-action@v3
+        with:
+          # Do not use local dir context to ensure we can build from a commit directly
+          # context: .
+          file: distributor-node.Dockerfile
+          push: false
+          load: true
+          tags: joystream/distributor-node:latest
+      - name: Build query-node
+        uses: docker/build-push-action@v3
+        with:
+          # Do not use local dir context to ensure we can build from a commit directly
+          # context: .
+          file: query-node.Dockerfile
+          push: false
+          load: true
+          tags: joystream/query-node:latest
+
+      - name: Test with latest images
+        run: |
+          yarn build:packages
+          cp docker-compose-no-bind-volumes.yml docker-compose.yml
+          ./tests/network-tests/run-tests.sh content-directory
+
+      - name: Push new versions
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          docker image tag joystream/storage-node:latest joystream/storage-node:${{ steps.extract_versions.outputs.colossus_version }}
+          docker image tag joystream/distributor-node:latest joystream/distributor-node:${{ steps.extract_versions.outputs.argus_version }}
+          docker image tag joystream/query-node:latest joystream/query-node:${{ steps.extract_versions.outputs.qn_version }}
+          docker push joystream/storage-node:${{ steps.extract_versions.outputs.colossus_version }}
+          docker push joystream/distributor-node:${{ steps.extract_versions.outputs.argus_version }}
+          docker push joystream/query-node:${{ steps.extract_versions.outputs.qn_version }}
+
+      - name: Push latest tag
+        if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch'
+        run: |
+          docker push joystream/storage-node:latest
+          docker push joystream/distributor-node:latest
+          docker push joystream/query-node:latest
+

.github/workflows/joystream-cli.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
       - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}
@@ -32,7 +32,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
       - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}

.github/workflows/joystream-node-docker-dev.yml

@@ -47,15 +47,36 @@ jobs:
           export IMAGE_EXISTS=$(docker manifest inspect ${{ env.REPOSITORY }}:${{ steps.compute_shasum.outputs.shasum }} > /dev/null ; echo $?)
           echo "::set-output name=image_exists::${IMAGE_EXISTS}"
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        # uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1.7
+        uses: docker/setup-buildx-action@v2
+        id: builder1
+        if: ${{ steps.compute_image_exists.outputs.image_exists == 1 }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-dev
+        if: ${{ steps.compute_image_exists.outputs.image_exists == 1 }}
+
       - name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           file: joystream-node.Dockerfile
+          # arm64 cross compiling takes longer than 6h, so we only build for amd64
+          # platforms: linux/amd64,linux/arm64
           platforms: linux/amd64
           build-args: |
             CARGO_FEATURES=${{ matrix.cargo_features }}
             CODE_SHASUM=${{ steps.compute_shasum.outputs.shasum }}
           push: true
           tags: ${{ env.REPOSITORY }}:${{ steps.compute_shasum.outputs.shasum }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          builder: ${{ steps.builder1.outputs.name }}
         if: ${{ steps.compute_image_exists.outputs.image_exists == 1 }}

.github/workflows/joystream-node-docker-publish.yml

@@ -18,7 +18,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
 
       - id: compute_shasum
         name: Compute runtime code shasum

.github/workflows/joystream-node-docker.yml

@@ -28,14 +28,21 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
 
       - id: compute_shasum
         name: Compute runtime code shasum
         run: |
           export RUNTIME_CODE_SHASUM=`scripts/runtime-code-shasum.sh`
           echo "::set-output name=shasum::${RUNTIME_CODE_SHASUM}"
 
+      # docker manifest inspect command requires authentication
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
       - name: Check if we already have the main image on Dockerhub
         id: compute_main_image_exists
         # Will output 0 if image exists and 1 if does not exists
@@ -71,8 +78,14 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
 
+      # docker manifest inspect requires authentication
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
       - name: Check if we have pre-built image on Dockerhub
         id: compute_image_exists
         # Will output 0 if image exists and 1 if does not exists
@@ -105,6 +118,10 @@ jobs:
           parameter-overrides: 'KeyName=${{ env.KEY_NAME }},EC2AMI=${{ matrix.ec2ami }},EC2InstanceType=${{ matrix.ec2type }}'
         if: ${{ steps.compute_image_exists.outputs.image_exists == 1 }}
 
+      - name: Wait for docker build server to be ready
+        run: |
+          sleep 30
+
       - name: Run playbook
         uses: dawidd6/action-ansible-playbook@v2
         with: