Merge pull request #17 from mnaamani/colossus-s3-api-integration-test…

…s-fix fix integration tests
Joystream · Aug 28, 2024 · 6e1f6e0 · 6e1f6e0
2 parents 5038ae4 + 5c7bc1a
commit 6e1f6e0
Show file tree

Hide file tree

Showing 19 changed files with 90 additions and 82 deletions.
diff --git a/docker-compose-no-bind-volumes.yml b/docker-compose-no-bind-volumes.yml
@@ -1,5 +1,4 @@
 # Complete joystream development network
-version: '3.4'
 services:
   joystream-node:
     image: joystream/node:$JOYSTREAM_NODE_TAG
@@ -37,6 +36,9 @@ services:
       - ACCOUNT_URI=${COLOSSUS_1_TRANSACTOR_URI}
       # - OTEL_EXPORTER_OTLP_ENDPOINT=http://apm-server:8200
       # - OTEL_RESOURCE_ATTRIBUTES=service.name=colossus-1,deployment.environment=production
+      - AWS_BUCKET_NAME=test-bucket-1
+      - LOCALSTACK_ENDPOINT=${LOCALSTACK_ENDPOINT}
+      - LOCALSTACK_ENABLED=${LOCALSTACK_ENABLED}
     entrypoint: ['yarn']
     command: [
       'start', '--worker=${COLOSSUS_1_WORKER_ID}', '--port=3333', '--uploads=/data/uploads',
@@ -106,6 +108,9 @@ services:
     environment:
       # ACCOUNT_URI overrides command line arg --accountUri
       - ACCOUNT_URI=${COLOSSUS_2_TRANSACTOR_URI}
+      - AWS_BUCKET_NAME=test-bucket-2
+      - LOCALSTACK_ENDPOINT=${LOCALSTACK_ENDPOINT}
+      - LOCALSTACK_ENABLED=${LOCALSTACK_ENABLED}
     entrypoint: ['yarn', 'storage-node']
     command: [
       'server', '--worker=${COLOSSUS_2_WORKER_ID}', '--port=3333', '--uploads=/data/uploads',

diff --git a/docker-compose.elasticsearch.yml b/docker-compose.elasticsearch.yml
@@ -1,5 +1,3 @@
-version: '3.4'
-
 services:
   # Ref: https://www.elastic.co/guide/en/elasticsearch/reference/8.7/docker.html
   elasticsearch:

diff --git a/docker-compose.localstack.yml b/docker-compose.localstack.yml
@@ -1,16 +1,12 @@
-version: '3.4'
 services:
   localstack:
     container_name: local_stack
-    image: localstack/localstack:3
+    image: localstack/localstack:3.3
     network_mode: bridge
     environment:
       - SERVICES=s3 # we only need s3 bucket
       - DEBUG=1
-      - DEFAULT_REGION=${AWS_REGION}
-      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
-      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
-      - LOCALSTACK_ENDPOINT=${LOCALSTACK_ENDPOINT}
+      - LOCALSTACK_HOST=${LOCALSTACK_HOST}
     ports:
       - 4566:4566
     volumes:

diff --git a/docker-compose.storage-squid.yml b/docker-compose.storage-squid.yml
@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   squid_db:
     container_name: squid_db

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,5 +1,4 @@
 # Complete joystream development network
-version: '3.4'
 services:
   joystream-node:
     image: joystream/node:$JOYSTREAM_NODE_TAG
@@ -37,6 +36,9 @@ services:
       - ACCOUNT_URI=${COLOSSUS_1_TRANSACTOR_URI}
       - OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT}
       - OTEL_RESOURCE_ATTRIBUTES=service.name=colossus-1,deployment.environment=production
+      - AWS_BUCKET_NAME=test-bucket-1
+      - LOCALSTACK_ENDPOINT=${LOCALSTACK_ENDPOINT}
+      - LOCALSTACK_ENABLED=${LOCALSTACK_ENABLED}
     entrypoint: ['/joystream/entrypoints/storage.sh']
     command:
       [
@@ -114,6 +116,9 @@ services:
     environment:
       # ACCOUNT_URI overrides command line arg --accountUri
       - ACCOUNT_URI=${COLOSSUS_2_TRANSACTOR_URI}
+      - AWS_BUCKET_NAME=test-bucket-2
+      - LOCALSTACK_ENDPOINT=${LOCALSTACK_ENDPOINT}
+      - LOCALSTACK_ENABLED=${LOCALSTACK_ENABLED}
     entrypoint: ['yarn', 'storage-node']
     command:
       [

diff --git a/query-node/kill.sh b/query-node/kill.sh
@@ -5,10 +5,10 @@ SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
 cd $SCRIPT_PATH
 
 # Only remove query-node related services
-docker-compose -f ../docker-compose.yml rm -vsf processor
-docker-compose -f ../docker-compose.yml rm -vsf graphql-server
-docker-compose -f ../docker-compose.yml rm -vsf indexer
-docker-compose -f ../docker-compose.yml rm -vsf hydra-indexer-gateway
-docker-compose -f ../docker-compose.yml rm -vsf redis
-docker-compose -f ../docker-compose.yml rm -vsf db
+docker compose -f ../docker-compose.yml rm -vsf processor
+docker compose -f ../docker-compose.yml rm -vsf graphql-server
+docker compose -f ../docker-compose.yml rm -vsf indexer
+docker compose -f ../docker-compose.yml rm -vsf hydra-indexer-gateway
+docker compose -f ../docker-compose.yml rm -vsf redis
+docker compose -f ../docker-compose.yml rm -vsf db
 docker volume rm joystream_query-node-data
diff --git a/query-node/reset-processor.sh b/query-node/reset-processor.sh
@@ -4,7 +4,7 @@ set -e
 SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")"
 cd $SCRIPT_PATH
 
-docker-compose -f ../docker-compose.yml rm -vsf processor
-docker-compose -f ../docker-compose.yml rm -vsf graphql-server
+docker compose -f ../docker-compose.yml rm -vsf processor
+docker compose -f ../docker-compose.yml rm -vsf graphql-server
 docker exec db psql -U postgres -c "DROP DATABASE query_node_processor;"
 ./start.sh
diff --git a/query-node/start.sh b/query-node/start.sh
@@ -8,22 +8,22 @@ cd $SCRIPT_PATH
 [ ! -d "generated/" ] && yarn build
 
 # Bring up db
-docker-compose -f ../docker-compose.yml up -d db
+docker compose -f ../docker-compose.yml up -d db
 echo "Waiting for the db to be ready..."
 sleep 5
 
 # Start indexer and gateway
-docker-compose -f ../docker-compose.yml up -d indexer
-docker-compose -f ../docker-compose.yml up -d hydra-indexer-gateway
+docker compose -f ../docker-compose.yml up -d indexer
+docker compose -f ../docker-compose.yml up -d hydra-indexer-gateway
 
 # Start processor
-docker-compose -f ../docker-compose.yml up -d processor
+docker compose -f ../docker-compose.yml up -d processor
 echo "Waiting for processor to be ready..." && sleep 30
 if [[ "$OSTYPE" == "darwin"* ]]; then
     # On Docker Desktop things take a bit longer to startup
     sleep 150
 fi
 
 # Start graphql-server
-docker-compose -f ../docker-compose.yml up -d graphql-server
+docker compose -f ../docker-compose.yml up -d graphql-server
 echo "Waiting for graphql-server to be ready..." && sleep 30
diff --git a/scripts/save-to-docker-images.sh b/scripts/save-to-docker-images.sh
@@ -2,10 +2,10 @@
 set -e
 
 # clean start
-docker-compose -f ../docker-compose.yml down -v
+docker compose -f ../docker-compose.yml down -v
 
 function cleanup() {
-    docker-compose -f ../docker-compose.yml down -v
+    docker compose -f ../docker-compose.yml down -v
 }
 
 trap cleanup EXIT
@@ -21,7 +21,7 @@ if [[ -z $JOYSTREAM_NODE_TAG ]]; then
 fi
 
 # start node image, network and volume
-docker-compose -f ../docker-compose.yml up -d joystream-node
+docker compose -f ../docker-compose.yml up -d joystream-node
 
 # copy native runtime
 docker cp ../target/release/joystream-node joystream-node:/joystream/node

diff --git a/start-elasticsearch-stack.sh b/start-elasticsearch-stack.sh
@@ -15,10 +15,10 @@ ELASTIC_USERNAME=${ELASTIC_USERNAME:="elastic"}
 ELASTIC_PASSWORD=${ELASTIC_PASSWORD:="password"}
 
 # Remove elasticsearch stack containers & volumes
-docker-compose -f ./docker-compose.elasticsearch.yml down -v
+docker compose -f ./docker-compose.elasticsearch.yml down -v
 
-# Run docker-compose to start elasticsearch container
-docker-compose -f ./docker-compose.elasticsearch.yml up -d elasticsearch
+# Run docker compose to start elasticsearch container
+docker compose -f ./docker-compose.elasticsearch.yml up -d elasticsearch
 
 echo 'Waiting for Elasticsearch...'
 
@@ -46,10 +46,10 @@ export ELASTICSEARCH_SERVICEACCOUNTTOKEN=$(echo $response_body | jq -r '.token.v
 
 echo 'Starting for Kibana...'
 
-## Run docker-compose to start kibana container
-docker-compose -f ./docker-compose.elasticsearch.yml up -d kibana
+## Run docker compose to start kibana container
+docker compose -f ./docker-compose.elasticsearch.yml up -d kibana
 
 echo 'Starting APM Server...'
 
-## Run docker-compose to start apm-server container
-docker-compose -f ./docker-compose.elasticsearch.yml up -d apm-server
+## Run docker compose to start apm-server container
+docker compose -f ./docker-compose.elasticsearch.yml up -d apm-server
diff --git a/start-multistorage.sh b/start-multistorage.sh
@@ -24,8 +24,8 @@ else
   function down()
   {
       # Stop containers and clear volumes
-      docker-compose -f ./docker-compose.storage-squid.yml down -v
-      docker-compose down -v
+      docker compose -f ./docker-compose.storage-squid.yml down -v
+      docker compose down -v
   }
 
   trap down EXIT ERR SIGINT SIGTERM
@@ -34,7 +34,7 @@ fi
 if [ "${SKIP_NODE}" != true ]
 then
   ## Run a local development chain
-  docker-compose up -d joystream-node
+  docker compose up -d joystream-node
 fi
 
 ## Query Node Infrastructure
@@ -44,7 +44,7 @@ fi
 ./start-orion.sh
 
 ## Storage Squid
-docker-compose -f ./docker-compose.storage-squid.yml up -d
+docker compose -f ./docker-compose.storage-squid.yml up -d
 
 ## Init the chain with some state
 if [[ $SKIP_CHAIN_SETUP != 'true' ]]; then
@@ -58,19 +58,19 @@ if [[ $SKIP_CHAIN_SETUP != 'true' ]]; then
 
   ## Member faucet
   export INVITER_KEY=`cat ./tests/network-tests/output.json | jq -r .faucet.suri`
-  docker-compose up -d faucet
+  docker compose up -d faucet
 
   ## Storage Infrastructure Nodes
-  docker-compose up -d colossus-1
-  docker-compose up -d distributor-1
-  docker-compose up -d colossus-2
-  docker-compose up -d distributor-2
+  docker compose up -d colossus-1
+  docker compose up -d distributor-1
+  docker compose up -d colossus-2
+  docker compose up -d distributor-2
 fi
 
 if [ "${PERSIST}" == true ]
 then
   echo "All services started in the background"
-  echo "Remember to run 'docker-compose down -v' to kill all docker services before starting new playground."
+  echo "Remember to run 'docker compose down -v' to kill all docker services before starting new playground."
 else
   echo "use Ctrl+C to shutdown the development network."
   while true; do

diff --git a/start-orion.sh b/start-orion.sh
@@ -1,11 +1,11 @@
 #!/usr/bin/env bash
 set -e
 
-docker-compose up -d orion-db
-docker-compose up -d orion-processor
-docker-compose up -d orion-graphql-api
-docker-compose up -d orion-auth-api
-docker-compose up -d squid-archive-ingest
-docker-compose up -d squid-archive-db
-docker-compose up -d squid-archive-gateway
-docker-compose up -d squid-archive-explorer
+docker compose up -d orion-db
+docker compose up -d orion-processor
+docker compose up -d orion-graphql-api
+docker compose up -d orion-auth-api
+docker compose up -d squid-archive-ingest
+docker compose up -d squid-archive-db
+docker compose up -d squid-archive-gateway
+docker compose up -d squid-archive-explorer
diff --git a/start.sh b/start.sh
@@ -26,8 +26,8 @@ else
   function down()
   {
       # Stop containers and clear volumes
-      docker-compose -f ./docker-compose.storage-squid.yml down -v
-      docker-compose down -v
+      docker compose -f ./docker-compose.storage-squid.yml down -v
+      docker compose down -v
   }
 
   trap down EXIT ERR SIGINT SIGTERM
@@ -36,7 +36,7 @@ fi
 if [ "${SKIP_NODE}" != true ]
 then
   ## Run a local development chain
-  docker-compose up -d joystream-node
+  docker compose up -d joystream-node
 fi
 
 ## Query Node Infrastructure
@@ -46,7 +46,7 @@ fi
 ./start-orion.sh
 
 ## Storage Squid
-docker-compose -f ./docker-compose.storage-squid.yml up -d
+docker compose -f ./docker-compose.storage-squid.yml up -d
 
 ## Init the chain with some state
 if [[ $SKIP_CHAIN_SETUP != true ]]; then
@@ -58,17 +58,17 @@ if [[ $SKIP_CHAIN_SETUP != true ]]; then
 
   ## Member faucet
   export INVITER_KEY=`cat ./tests/network-tests/output.json | jq -r .faucet.suri`
-  docker-compose up -d faucet
+  docker compose up -d faucet
 
   ## Storage Infrastructure Nodes
-  docker-compose up -d colossus-1
-  docker-compose up -d distributor-1
+  docker compose up -d colossus-1
+  docker compose up -d distributor-1
 fi
 
 if [ "${PERSIST}" == true ]
 then
   echo "All services started in the background"
-  echo "Remember to run 'docker-compose down -v' to kill all docker services before starting new playground."
+  echo "Remember to run 'docker compose down -v' to kill all docker services before starting new playground."
 else
   echo "use Ctrl+C to shutdown the development network."
   while true; do

diff --git a/tests/network-tests/.gitignore b/tests/network-tests/.gitignore
@@ -1,3 +1,4 @@
 output.json
 data/
 **/generated/*
+.venv/
diff --git a/tests/network-tests/run-node-docker.sh b/tests/network-tests/run-node-docker.sh
@@ -66,7 +66,7 @@ docker run --pull never --rm -v ${DATA_PATH}:/spec joystream/node:${RUNTIME} bui
 
 # Start a chain with generated chain spec
 export JOYSTREAM_NODE_TAG=${RUNTIME}
-docker-compose -p joystream -f ../../docker-compose.yml run -d -v ${DATA_PATH}:/spec --name joystream-node \
+docker compose -p joystream -f ../../docker-compose.yml run -d -v ${DATA_PATH}:/spec --name joystream-node \
   --service-ports joystream-node \
   --alice --validator --unsafe-ws-external --unsafe-rpc-external \
   --rpc-methods Unsafe --rpc-cors=all -l runtime \

diff --git a/tests/network-tests/run-runtime-upgrade-tests.sh b/tests/network-tests/run-runtime-upgrade-tests.sh
@@ -68,7 +68,7 @@ function create_raw_chain_spec() {
 
 # Start a chain with generated chain spec
 function start_joystream_node {
-    docker-compose -f ../../docker-compose.yml run -d -v ${DATA_PATH}:/spec \
+    docker compose -f ../../docker-compose.yml run -d -v ${DATA_PATH}:/spec \
         --name joystream-node \
         -p 9944:9944 -p 9933:9933 joystream-node \
         --validator --unsafe-ws-external --unsafe-rpc-external \
@@ -140,7 +140,7 @@ function init_chain_db() {
     # if the initial state is large.
     # exporting should give some essential tasks errors but they are harmless https://github.com/paritytech/substrate/issues/10583
     echo >&2 "exporting state"
-    docker-compose -f ../../docker-compose.yml run --rm \
+    docker compose -f ../../docker-compose.yml run --rm \
         -v ${DATA_PATH}:/spec joystream-node export-state \
         --chain /spec/chain-spec-forked.json \
         --base-path /data --pruning archive >${DATA_PATH}/exported-state.json

diff --git a/tests/network-tests/run-tests.sh b/tests/network-tests/run-tests.sh
@@ -16,17 +16,16 @@ function cleanup() {
   docker logs colossus-2 --tail 100 || :
 
   if [ "${NO_STORAGE}" != true ]; then
-    docker-compose -f ../../docker-compose.storage-squid.yml down -v
-    docker-compose -f ../../docker-compose.localstack.yml down -v
+    docker compose -f ../../docker-compose.storage-squid.yml down -v
+    docker compose -f ../../docker-compose.localstack.yml down -v
   fi
 
-  docker-compose -f ../../docker-compose.yml down -v
+  docker compose -f ../../docker-compose.yml down -v
 }
 
 trap cleanup EXIT ERR SIGINT SIGTERM
 
-export JOYSTREAM_NODE_TAG=${JOYSTREAM_NODE_TAG:-latest}
-RUNTIME_PROFILE=TESTING ../../scripts/runtime-code-shasum.sh
+export JOYSTREAM_NODE_TAG=`RUNTIME_PROFILE=TESTING ../../scripts/runtime-code-shasum.sh`
 CHAIN=dev docker compose -f ../../docker-compose.yml up -d joystream-node
 
 sleep 30