Merge branch 'docker-compose-orion' into crt-release

.dockerignore

@@ -4,8 +4,6 @@ target/
 .vscode/
 query-node/**/dist
 query-node/lib
-tests/
-!tests/network-tests/proposal-parameters.json
 devops/
 !devops/eslint-config
-!devops/prettier-config
+!devops/prettier-config

.env

@@ -1,10 +1,11 @@
 COMPOSE_PROJECT_NAME=joystream
 PROJECT_NAME=query_node
 
-# We will use a single postgres service with multiple databases
-# The env variables below are by default used by all services and should be
-# overriden in local env files
-# DB config
+# We use a single postgres service for both the query node indexer and processor.
+# The default `DB_*` environment variables point to query node processor's database.
+# The DB_NAME env is overriden with INDEXER_DB_NAME in the indexer's service docker-compose config.
+# When running other services, those values should be overriden by their local environment configs.
+# Query node DB config:
 INDEXER_DB_NAME=query_node_indexer
 DB_NAME=query_node_processor
 DB_USER=postgres
@@ -90,6 +91,80 @@ HCAPTCHA_SECRET=
 BALANCE_CREDIT=300000000000
 BALANCE_LOCKED=300000000000
 
+# ====================================== ORION ======================================
+# LOCAL DEV ENVIRONMENT
+
+ORION_ENV=development
+DEV_DISABLE_SAME_SITE=true
+
+# Db config
+ORION_DB_NAME=squid
+ORION_DB_PASS=squid
+DB_ADMIN_USER=admin
+DB_ADMIN_PASS=admin
+ORION_DB_PORT=5433
+
+# archive configuration
+ARCHIVE_DB_PORT=12345
+ARCHIVE_GATEWAY_PORT=8888
+WS_SOURCE=ws://joystream-node:9944
+ARCHIVE_EXPLORER_PORT=4444
+
+# Processor service prometheus port
+PROCESSOR_PROMETHEUS_PORT=3337
+# Graphql server port
+GRAPHQL_API_PORT=4350
+# Auth api port
+AUTH_API_PORT=4074
+
+APP_NAME=Gleev
+SUPPORT_NO_CATEGORY_VIDEOS=true
+SUPPORT_NEW_CATEGORIES=true
+KILL_SWITCH_ON=false
+# 10 seconds
+VIDEO_VIEW_PER_USER_TIME_LIMIT=10
+# Operator API secret
+OPERATOR_SECRET=this-is-not-so-secret-change-it
+# every 50 views video relevance score will be recalculated
+VIDEO_RELEVANCE_VIEWS_TICK=50
+# [
+#   newness (negative number of days since created) weight,
+#   views weight,
+#   comments weight,
+#   rections weights,
+#   [joystream creation weight, YT creation weight]
+# ]
+RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5, [7,3]]"
+MAX_CACHED_ENTITIES=1000
+APP_PRIVATE_KEY=this-is-not-so-secret-change-it
+SESSION_EXPIRY_AFTER_INACTIVITY_MINUTES=60
+SESSION_MAX_DURATION_HOURS=720
+EMAIL_CONFIRMATION_ROUTE=http://localhost:4074/api/v1/confirm-email?token={token}
+EMAIL_CONFIRMATION_TOKEN_EXPIRY_TIME_HOURS=24
+EMAIL_CONFIRMATION_TOKEN_RATE_LIMIT=5
+ACCOUNT_OWNERSHIP_PROOF_EXPIRY_TIME_SECONDS=300 # 5 minutes
+COOKIE_SECRET=this-is-not-so-secret-change-it
+
+TRUST_PROXY=uniquelocal
+
+# Sendgrid API
+SENDGRID_API_KEY=
+SENDGRID_FROM_EMAIL=[email protected]
+
+# Debug settings
+SQD_DEBUG=api:*
+OPENAPI_PLAYGROUND=true
+
+ARCHIVE_GATEWAY_URL=${CUSTOM_ARCHIVE_GATEWAY_URL:-http://squid-archive-gateway:8000/graphql}
+
+# =====================================================================================
+
+## Telemetry
+# yes/no
+TELEMETRY_ENABLED=no
+# TELEMETRY_ENDPOINT=http://apm-server:8200
+TELEMETRY_ENDPOINT=http://collector:4318
+
 # joystream/node docker image tag
 # We do not provide a default value - scripts that startup a joystream-node service
 # Should be explicit about what version to use.

.github/workflows/deploy-node-network.yml

@@ -86,6 +86,10 @@ jobs:
             EC2InstanceType=${{ steps.network_config.outputs.instanceType }},
             VolumeSize=${{ steps.network_config.outputs.volumeSize }}
 
+      - name: Wait for docker build server to be ready
+        run: |
+          sleep 30
+
       - name: Prepare inventory for Ansible
         run: |
           VAL1="${{ steps.deploy_stack.outputs.Val1PublicIp }}"

.github/workflows/deploy-playground.yml

@@ -86,6 +86,10 @@ jobs:
           # Make sure ami image is available in the region specified in configure aws creds step
           parameter-overrides: 'KeyName=joystream-github-action-key-new,EC2InstanceType=t2.xlarge,EC2AMI=ami-06b4d9ba1f23a8da4'
 
+      - name: Wait for docker build server to be ready
+        run: |
+          sleep 30
+
       - name: Run playbook
         uses: dawidd6/action-ansible-playbook@v2
         with:

.github/workflows/joystream-apps-docker.yml

@@ -1,30 +1,103 @@
-name: Build joystream/apps and publish to Docker Hub
+name: Publish Colossus/Argus/QueryNode Docker images
 
 on:
   workflow_dispatch:
-    inputs:
-      tag_suffix:
-        description: 'Tag suffix'
-        required: true
+  pull_request:
+
 jobs:
-  docker:
+  build_and_publish:
+    name: Build and Publish
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
-      - name: Extract branch name
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+
+      - name: Extract Package Versions
+        id: extract_versions
         shell: bash
-        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: extract_branch
+        run: |
+          echo "colossus_version=$(cat storage-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
+          echo "argus_version=$(cat distributor-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
+          echo "qn_version=$(cat query-node/package.json | jq -r '.version')" >> $GITHUB_OUTPUT
+
+      - name: Make some space
+        shell: bash
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
+        if: github.event_name == 'workflow_dispatch'
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
-      - name: Build and push
-        id: docker_build
+
+      - name: Build joystream-node
+        run: RUNTIME_PROFILE=TESTING ./build-node-docker.sh
+
+      # docker/build-push-action doc:
+      # Be careful because any file mutation in the steps that precede the
+      # build step will be ignored, including processing of the .dockerignore file
+      # since the context is based on the Git reference. However, you can use
+      # the Path context using the context input alongside the actions/checkout action
+      # to remove this restriction.
+      - name: Build storage-node
         uses: docker/build-push-action@v3
         with:
-          file: apps.Dockerfile
-          push: true
-          tags: joystream/apps:${{ steps.extract_branch.outputs.branch }}-${{ github.event.inputs.tag_suffix }}
+          # Do not use local dir context to ensure we can build from a commit directly
+          # context: .
+          file: colossus.Dockerfile
+          push: false
+          load: true
+          tags: joystream/storage-node:latest
+      - name: Build distributor-node
+        uses: docker/build-push-action@v3
+        with:
+          # Do not use local dir context to ensure we can build from a commit directly
+          # context: .
+          file: distributor-node.Dockerfile
+          push: false
+          load: true
+          tags: joystream/distributor-node:latest
+      - name: Build query-node
+        uses: docker/build-push-action@v3
+        with:
+          # Do not use local dir context to ensure we can build from a commit directly
+          # context: .
+          file: query-node.Dockerfile
+          push: false
+          load: true
+          tags: joystream/query-node:latest
+
+      - name: Test with latest images
+        run: |
+          yarn build:packages
+          cp docker-compose-no-bind-volumes.yml docker-compose.yml
+          ./tests/network-tests/run-tests.sh content-directory
+
+      - name: Push new versions
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          docker image tag joystream/storage-node:latest joystream/storage-node:${{ steps.extract_versions.outputs.colossus_version }}
+          docker image tag joystream/distributor-node:latest joystream/distributor-node:${{ steps.extract_versions.outputs.argus_version }}
+          docker image tag joystream/query-node:latest joystream/query-node:${{ steps.extract_versions.outputs.qn_version }}
+          docker push joystream/storage-node:${{ steps.extract_versions.outputs.colossus_version }}
+          docker push joystream/distributor-node:${{ steps.extract_versions.outputs.argus_version }}
+          docker push joystream/query-node:${{ steps.extract_versions.outputs.qn_version }}
+
+      - name: Push latest tag
+        if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch'
+        run: |
+          docker push joystream/storage-node:latest
+          docker push joystream/distributor-node:latest
+          docker push joystream/query-node:latest
+

.github/workflows/joystream-cli.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
       - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}
@@ -32,7 +32,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
       - uses: actions/checkout@v3
       - name: Use Node.js ${{ matrix.node-version }}

.github/workflows/joystream-node-docker-publish.yml

@@ -18,7 +18,7 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
 
       - id: compute_shasum
         name: Compute runtime code shasum

.github/workflows/joystream-node-docker.yml

@@ -28,14 +28,21 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
 
       - id: compute_shasum
         name: Compute runtime code shasum
         run: |
           export RUNTIME_CODE_SHASUM=`scripts/runtime-code-shasum.sh`
           echo "::set-output name=shasum::${RUNTIME_CODE_SHASUM}"
 
+      # docker manifest inspect command requires authentication
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
       - name: Check if we already have the main image on Dockerhub
         id: compute_main_image_exists
         # Will output 0 if image exists and 1 if does not exists
@@ -71,8 +78,14 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
 
+      # docker manifest inspect requires authentication
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
       - name: Check if we have pre-built image on Dockerhub
         id: compute_image_exists
         # Will output 0 if image exists and 1 if does not exists
@@ -105,6 +118,10 @@ jobs:
           parameter-overrides: 'KeyName=${{ env.KEY_NAME }},EC2AMI=${{ matrix.ec2ami }},EC2InstanceType=${{ matrix.ec2type }}'
         if: ${{ steps.compute_image_exists.outputs.image_exists == 1 }}
 
+      - name: Wait for docker build server to be ready
+        run: |
+          sleep 30
+
       - name: Run playbook
         uses: dawidd6/action-ansible-playbook@v2
         with:

.github/workflows/joystream-node.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
       - uses: technote-space/get-diff-action@v3
         with:
           PREFIX_FILTER: |
@@ -56,7 +56,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-node@v3
         with:
-          node-version: '14.x'
+          node-version: '18.x'
       - uses: technote-space/get-diff-action@v3
         with:
           PREFIX_FILTER: |

.github/workflows/joystream-types.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}
@@ -30,7 +30,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}

.github/workflows/lint-typescript.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}
@@ -24,7 +24,7 @@ jobs:
     runs-on: macos-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}

.github/workflows/metadata-protobuf.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [14.x]
+        node-version: [18.x]
     steps:
     - uses: actions/checkout@v3
     - name: Use Node.js ${{ matrix.node-version }}