diff --git a/.env b/.env index 13e8c403b8..cff3841258 100644 --- a/.env +++ b/.env @@ -92,35 +92,71 @@ BALANCE_CREDIT=300000000000 BALANCE_LOCKED=300000000000 # ====================================== ORION ====================================== -# Orion db config -# Overrides `DB_*` environment variables for orion services in docker-compose.yml -# We use a single postgres service for both: -# - Subsquid archive's database (squid-archive, squid-archive-gateway) and -# - Orion's database (orion_processor, orion_graphql-server). -# ORION_DB_NAME is the name of the Orion's database, while SQUID_ARCHIVE_DB_NAME -# is the name of the Subsquid archive database. -SQUID_ARCHIVE_DB_NAME=squid_archive -ORION_DB_NAME=orion -ORION_DB_USER=postgres -ORION_DB_PASS=postgres +# LOCAL DEV ENVIRONMENT + +ORION_ENV=development +DEV_DISABLE_SAME_SITE=true + +# Db config +ORION_DB_NAME=squid +ORION_DB_PASS=squid +DB_ADMIN_USER=admin +DB_ADMIN_PASS=admin ORION_DB_PORT=5433 -# Orion / Subsquid archive service ports -ORION_API_PORT=4350 +# archive configuration +ARCHIVE_DB_PORT=12345 +ARCHIVE_GATEWAY_PORT=8888 +WS_SOURCE=ws://joystream-node:9944 +ARCHIVE_EXPLORER_PORT=4444 + +# Processor service prometheus port PROCESSOR_PROMETHEUS_PORT=3337 -SQUID_ARCHIVE_GATEWAY_PORT=8000 +# Graphql server port +GRAPHQL_API_PORT=4350 +# Auth api port +AUTH_API_PORT=4074 -# Other Orion configs +APP_NAME=Gleev SUPPORT_NO_CATEGORY_VIDEOS=true SUPPORT_NEW_CATEGORIES=true KILL_SWITCH_ON=false -VIDEO_VIEW_PER_IP_TIME_LIMIT=30 -VIDEO_RELEVANCE_VIEWS_TICK=50 -RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5]" +# 10 seconds +VIDEO_VIEW_PER_USER_TIME_LIMIT=10 +# Operator API secret OPERATOR_SECRET=this-is-not-so-secret-change-it +# every 50 views video relevance score will be recalculated +VIDEO_RELEVANCE_VIEWS_TICK=50 +# [ +# newness (negative number of days since created) weight, +# views weight, +# comments weight, +# rections weights, +# [joystream creation weight, YT creation weight] +# ] +RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5, [7,3]]" MAX_CACHED_ENTITIES=1000 APP_PRIVATE_KEY=this-is-not-so-secret-change-it -TRUSTED_REVERSE_PROXIES=1 +SESSION_EXPIRY_AFTER_INACTIVITY_MINUTES=60 +SESSION_MAX_DURATION_HOURS=720 +EMAIL_CONFIRMATION_ROUTE=http://localhost:4074/api/v1/confirm-email?token={token} +EMAIL_CONFIRMATION_TOKEN_EXPIRY_TIME_HOURS=24 +EMAIL_CONFIRMATION_TOKEN_RATE_LIMIT=5 +ACCOUNT_OWNERSHIP_PROOF_EXPIRY_TIME_SECONDS=300 # 5 minutes +COOKIE_SECRET=this-is-not-so-secret-change-it + +TRUST_PROXY=uniquelocal + +# Sendgrid API +ORION_SENDGRID_API_KEY= +ORION_SENDGRID_FROM_EMAIL=gateway@example.com + +# Debug settings +SQD_DEBUG=api:* +OPENAPI_PLAYGROUND=true + +ARCHIVE_GATEWAY_URL=${CUSTOM_ARCHIVE_GATEWAY_URL:-http://squid-archive-gateway:8000/graphql} + # ===================================================================================== ## Telemetry diff --git a/docker-compose-no-bind-volumes.yml b/docker-compose-no-bind-volumes.yml index 40222a7ebf..87f6d0d7d8 100644 --- a/docker-compose-no-bind-volumes.yml +++ b/docker-compose-no-bind-volumes.yml @@ -164,23 +164,6 @@ services: POSTGRES_DB: ${INDEXER_DB_NAME} PGPORT: ${DB_PORT} - orion-db: - image: postgres:14 - container_name: orion-db - restart: unless-stopped - ports: - - '127.0.0.1:${ORION_DB_PORT}:${ORION_DB_PORT}' - volumes: - - orion-db-data:/var/lib/postgresql/data - env_file: - # relative to working directory where docker-compose was run from - - .env - environment: - POSTGRES_USER: ${ORION_DB_USER} - POSTGRES_PASSWORD: ${ORION_DB_PASS} - POSTGRES_DB: ${ORION_DB_NAME} - PGPORT: ${ORION_DB_PORT} - graphql-server: image: joystream/query-node build: @@ -307,92 +290,162 @@ services: ports: - "3002:3002" - squid-archive: - container_name: squid-archive - image: subsquid/substrate-ingest:firesquid - depends_on: - - orion-db + orion-db: + image: postgres:14 + container_name: orion-db restart: unless-stopped env_file: - .env environment: - - DB_NAME=${SUBSQUID_ARCHIVE_DB_NAME} - - DB_USER=${ORION_DB_USER} - - DB_PASS=${ORION_DB_PASS} - - DB_HOST=orion-db - - DB_PORT=${ORION_DB_PORT} - command: [ - '-e', - '${JOYSTREAM_NODE_WS}', - '-c', - '20', - '--out', - 'postgres://postgres:postgres@orion-db:${ORION_DB_PORT}/${SUBSQUID_ARCHIVE_DB_NAME}', - ] + POSTGRES_PASSWORD: ${ORION_DB_PASS} + POSTGRES_DB: ${ORION_DB_NAME} + ports: + - '127.0.0.1:${ORION_DB_PORT}:${ORION_DB_PORT}' + command: ['postgres', '-c', 'config_file=/etc/postgresql/postgresql.conf', '-p', '${ORION_DB_PORT}'] + # command: ['postgres', '-p', '${ORION_DB_PORT}'] + shm_size: 1g + volumes: + - orion-db-data:/var/lib/postgresql/data + - ./orion-postgres.conf:/etc/postgresql/postgresql.conf - squid-archive-gateway: - container_name: squid-archive-gateway - image: subsquid/substrate-gateway:firesquid + # Orion processor + orion-processor: + image: joystream/orion:latest + container_name: orion-processor + restart: unless-stopped depends_on: - orion-db - restart: unless-stopped env_file: - .env environment: - - DB_NAME=${SUBSQUID_ARCHIVE_DB_NAME} - - DB_USER=${ORION_DB_USER} + - DB_NAME=${ORION_DB_NAME} - DB_PASS=${ORION_DB_PASS} - - DB_HOST=orion-db - DB_PORT=${ORION_DB_PORT} - - DATABASE_MAX_CONNECTIONS=5 - - RUST_LOG=actix_web=info,actix_server=info - command: [ - '--database-url', - 'postgres://postgres:postgres@orion-db:${ORION_DB_PORT}/${SUBSQUID_ARCHIVE_DB_NAME}', - ] + - DB_HOST=orion-db + - GQL_PORT=${GRAPHQL_API_PORT} + volumes: + - ./db/export:/squid/db/export + command: > + sh -c "npx squid-typeorm-migration apply && \ + node lib/processor.js" ports: - - '127.0.0.1:${SQUID_ARCHIVE_GATEWAY_PORT}:8000' + - '127.0.0.1:${PROCESSOR_PROMETHEUS_PORT}:${PROCESSOR_PROMETHEUS_PORT}' - orion-processor: - container_name: orion-processor + # Orion GraphQL API + orion-graphql-api: + container_name: orion-graphql-api image: joystream/orion:latest restart: unless-stopped - env_file: - - .env depends_on: - orion-db + env_file: + - .env environment: - DB_NAME=${ORION_DB_NAME} - - DB_USER=${ORION_DB_USER} - DB_PASS=${ORION_DB_PASS} - - DB_HOST=orion-db - DB_PORT=${ORION_DB_PORT} - - GQL_PORT=${ORION_API_PORT} - - ARCHIVE_GATEWAY_URL=http://squid-archive-gateway:8000/graphql - command: > - sh -c "npx squid-typeorm-migration apply && \ - node lib/processor.js" + - DB_HOST=orion-db + - GQL_PORT=${GRAPHQL_API_PORT} + command: ['npm', 'run', 'query-node-start'] ports: - - '127.0.0.1:${PROCESSOR_PROMETHEUS_PORT}:${PROCESSOR_PROMETHEUS_PORT}' + - '127.0.0.1:${GRAPHQL_API_PORT}:${GRAPHQL_API_PORT}' - orion-api: - container_name: orion-api + # Orion Authentication API + orion-auth-api: + container_name: orion-auth-api image: joystream/orion:latest restart: unless-stopped - env_file: - - .env depends_on: - orion-db + env_file: + - .env environment: - DB_NAME=${ORION_DB_NAME} - - DB_USER=${ORION_DB_USER} - DB_PASS=${ORION_DB_PASS} - - DB_HOST=orion-db - DB_PORT=${ORION_DB_PORT} - - GQL_PORT=${ORION_API_PORT} - command: ["npm", "run", "query-node-start"] + - DB_HOST=orion-db + - SENDGRID_API_KEY=${ORION_SENDGRID_API_KEY} + - SENDGRID_FROM_EMAIL=${ORION_SENDGRID_FROM_EMAIL} + command: ['npm', 'run', 'auth-server-start'] + ports: + - '127.0.0.1:${AUTH_API_PORT}:${AUTH_API_PORT}' + + # Orion archive node + squid-archive-db: + container_name: orion_archive_db + hostname: orion_archive_db + image: postgres:14 # CockroachDB cluster might be a better fit for production deployment + restart: unless-stopped + volumes: + - orion_archive_db_data:/var/lib/postgresql/data + env_file: + - .env + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: squid-archive + ports: + - '127.0.0.1:${ARCHIVE_DB_PORT}:${ARCHIVE_DB_PORT}' + - '[::1]:${ARCHIVE_DB_PORT}:${ARCHIVE_DB_PORT}' + command: ['postgres', '-c', 'log_statement=all', '-p', '${ARCHIVE_DB_PORT}'] + + squid-archive-ingest: + container_name: orion_archive_ingest + depends_on: + - squid-archive-db + restart: unless-stopped + env_file: + - .env + image: subsquid/substrate-ingest:firesquid + command: [ + '-e', + '${WS_SOURCE}', + '-c', + '20', # allow up to 20 pending requests for the above endpoint (default is 5) + # "--start-block", "1000000", # uncomment to specify a non-zero start block + '--out', + 'postgres://postgres:postgres@orion_archive_db:${ARCHIVE_DB_PORT}/squid-archive', + ] + + squid-archive-gateway: + container_name: orion_archive_gateway + hostname: orion_archive_gateway + depends_on: + - squid-archive-db + restart: unless-stopped + image: subsquid/substrate-gateway:firesquid + env_file: + - .env + environment: + DATABASE_MAX_CONNECTIONS: 5 + RUST_LOG: 'actix_web=info,actix_server=info' + command: [ + '--database-url', + 'postgres://postgres:postgres@orion_archive_db:${ARCHIVE_DB_PORT}/squid-archive', + ] ports: - - '${ORION_API_PORT}:${ORION_API_PORT}' + - '127.0.0.1:${ARCHIVE_GATEWAY_PORT}:8000' + - '[::1]:${ARCHIVE_GATEWAY_PORT}:8000' + + # Explorer service is optional. + # It provides rich GraphQL API for querying archived data. + # Many developers find it very useful for exploration and debugging. + squid-archive-explorer: + container_name: orion_archive_explorer + hostname: orion_archive_explorer + restart: unless-stopped + image: subsquid/substrate-explorer:firesquid + env_file: + - .env + environment: + DB_TYPE: postgres # set to `cockroach` for Cockroach DB + DB_HOST: orion_archive_db + DB_NAME: 'squid-archive' + DB_USER: 'postgres' + DB_PASS: 'postgres' + DB_PORT: ${ARCHIVE_DB_PORT} + ports: + - '${ARCHIVE_EXPLORER_PORT}:3000' volumes: chain-data: @@ -425,3 +478,5 @@ volumes: driver: local orion-db-data: driver: local + orion_archive_db_data: + driver: local diff --git a/docker-compose.yml b/docker-compose.yml index b00f28a242..dd3f6e9348 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -170,23 +170,6 @@ services: POSTGRES_DB: ${INDEXER_DB_NAME} PGPORT: ${DB_PORT} - orion-db: - image: postgres:14 - container_name: orion-db - restart: unless-stopped - ports: - - '127.0.0.1:${ORION_DB_PORT}:${ORION_DB_PORT}' - volumes: - - orion-db-data:/var/lib/postgresql/data - env_file: - # relative to working directory where docker-compose was run from - - .env - environment: - POSTGRES_USER: ${ORION_DB_USER} - POSTGRES_PASSWORD: ${ORION_DB_PASS} - POSTGRES_DB: ${ORION_DB_NAME} - PGPORT: ${ORION_DB_PORT} - graphql-server: image: node:18 container_name: graphql-server @@ -318,92 +301,163 @@ services: ports: - "3002:3002" - squid-archive: - container_name: squid-archive - image: subsquid/substrate-ingest:firesquid - depends_on: - - orion-db + # PostgerSQL database for Orion + orion-db: + image: postgres:14 + container_name: orion-db restart: unless-stopped env_file: - .env environment: - - DB_NAME=${SUBSQUID_ARCHIVE_DB_NAME} - - DB_USER=${ORION_DB_USER} - - DB_PASS=${ORION_DB_PASS} - - DB_HOST=orion-db - - DB_PORT=${ORION_DB_PORT} - command: [ - '-e', - '${JOYSTREAM_NODE_WS}', - '-c', - '20', - '--out', - 'postgres://postgres:postgres@orion-db:${ORION_DB_PORT}/${SUBSQUID_ARCHIVE_DB_NAME}', - ] + POSTGRES_PASSWORD: ${ORION_DB_PASS} + POSTGRES_DB: ${ORION_DB_NAME} + ports: + - '127.0.0.1:${ORION_DB_PORT}:${ORION_DB_PORT}' + command: ['postgres', '-c', 'config_file=/etc/postgresql/postgresql.conf', '-p', '${ORION_DB_PORT}'] + # command: ['postgres', '-p', '${ORION_DB_PORT}'] + shm_size: 1g + volumes: + - orion-db-data:/var/lib/postgresql/data + - ./orion-postgres.conf:/etc/postgresql/postgresql.conf - squid-archive-gateway: - container_name: squid-archive-gateway - image: subsquid/substrate-gateway:firesquid + # Orion processor + orion-processor: + image: joystream/orion:latest + container_name: orion-processor + restart: unless-stopped depends_on: - orion-db - restart: unless-stopped env_file: - .env environment: - - DB_NAME=${SUBSQUID_ARCHIVE_DB_NAME} - - DB_USER=${ORION_DB_USER} + - DB_NAME=${ORION_DB_NAME} - DB_PASS=${ORION_DB_PASS} - - DB_HOST=orion-db - DB_PORT=${ORION_DB_PORT} - - DATABASE_MAX_CONNECTIONS=5 - - RUST_LOG=actix_web=info,actix_server=info - command: [ - '--database-url', - 'postgres://postgres:postgres@orion-db:${ORION_DB_PORT}/${SUBSQUID_ARCHIVE_DB_NAME}', - ] + - DB_HOST=orion-db + - GQL_PORT=${GRAPHQL_API_PORT} + volumes: + - ./db/export:/squid/db/export + command: > + sh -c "npx squid-typeorm-migration apply && \ + node lib/processor.js" ports: - - '127.0.0.1:${SQUID_ARCHIVE_GATEWAY_PORT}:8000' + - '127.0.0.1:${PROCESSOR_PROMETHEUS_PORT}:${PROCESSOR_PROMETHEUS_PORT}' - orion-processor: - container_name: orion-processor + # Orion GraphQL API + orion-graphql-api: + container_name: orion-graphql-api image: joystream/orion:latest restart: unless-stopped - env_file: - - .env depends_on: - orion-db + env_file: + - .env environment: - DB_NAME=${ORION_DB_NAME} - - DB_USER=${ORION_DB_USER} - DB_PASS=${ORION_DB_PASS} - - DB_HOST=orion-db - DB_PORT=${ORION_DB_PORT} - - GQL_PORT=${ORION_API_PORT} - - ARCHIVE_GATEWAY_URL=http://squid-archive-gateway:8000/graphql - command: > - sh -c "npx squid-typeorm-migration apply && \ - node lib/processor.js" + - DB_HOST=orion-db + - GQL_PORT=${GRAPHQL_API_PORT} + command: ['npm', 'run', 'query-node-start'] ports: - - '127.0.0.1:${PROCESSOR_PROMETHEUS_PORT}:${PROCESSOR_PROMETHEUS_PORT}' + - '127.0.0.1:${GRAPHQL_API_PORT}:${GRAPHQL_API_PORT}' - orion-api: - container_name: orion-api + # Orion Authentication API + orion-auth-api: + container_name: orion-auth-api image: joystream/orion:latest restart: unless-stopped - env_file: - - .env depends_on: - orion-db + env_file: + - .env environment: - DB_NAME=${ORION_DB_NAME} - - DB_USER=${ORION_DB_USER} - DB_PASS=${ORION_DB_PASS} - - DB_HOST=orion-db - DB_PORT=${ORION_DB_PORT} - - GQL_PORT=${ORION_API_PORT} - command: ["npm", "run", "query-node-start"] + - DB_HOST=orion-db + - SENDGRID_API_KEY=${ORION_SENDGRID_API_KEY} + - SENDGRID_FROM_EMAIL=${ORION_SENDGRID_FROM_EMAIL} + command: ['npm', 'run', 'auth-server-start'] + ports: + - '127.0.0.1:${AUTH_API_PORT}:${AUTH_API_PORT}' + + # Orion archive node + squid-archive-db: + container_name: orion_archive_db + hostname: orion_archive_db + image: postgres:14 # CockroachDB cluster might be a better fit for production deployment + restart: unless-stopped + volumes: + - orion_archive_db_data:/var/lib/postgresql/data + env_file: + - .env + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: squid-archive + ports: + - '127.0.0.1:${ARCHIVE_DB_PORT}:${ARCHIVE_DB_PORT}' + - '[::1]:${ARCHIVE_DB_PORT}:${ARCHIVE_DB_PORT}' + command: ['postgres', '-c', 'log_statement=all', '-p', '${ARCHIVE_DB_PORT}'] + + squid-archive-ingest: + container_name: orion_archive_ingest + depends_on: + - squid-archive-db + restart: unless-stopped + env_file: + - .env + image: subsquid/substrate-ingest:firesquid + command: [ + '-e', + '${WS_SOURCE}', + '-c', + '20', # allow up to 20 pending requests for the above endpoint (default is 5) + # "--start-block", "1000000", # uncomment to specify a non-zero start block + '--out', + 'postgres://postgres:postgres@orion_archive_db:${ARCHIVE_DB_PORT}/squid-archive', + ] + + squid-archive-gateway: + container_name: orion_archive_gateway + hostname: orion_archive_gateway + depends_on: + - squid-archive-db + restart: unless-stopped + image: subsquid/substrate-gateway:firesquid + env_file: + - .env + environment: + DATABASE_MAX_CONNECTIONS: 5 + RUST_LOG: 'actix_web=info,actix_server=info' + command: [ + '--database-url', + 'postgres://postgres:postgres@orion_archive_db:${ARCHIVE_DB_PORT}/squid-archive', + ] ports: - - '${ORION_API_PORT}:${ORION_API_PORT}' + - '127.0.0.1:${ARCHIVE_GATEWAY_PORT}:8000' + - '[::1]:${ARCHIVE_GATEWAY_PORT}:8000' + + # Explorer service is optional. + # It provides rich GraphQL API for querying archived data. + # Many developers find it very useful for exploration and debugging. + squid-archive-explorer: + container_name: orion_archive_explorer + hostname: orion_archive_explorer + restart: unless-stopped + image: subsquid/substrate-explorer:firesquid + env_file: + - .env + environment: + DB_TYPE: postgres # set to `cockroach` for Cockroach DB + DB_HOST: orion_archive_db + DB_NAME: 'squid-archive' + DB_USER: 'postgres' + DB_PASS: 'postgres' + DB_PORT: ${ARCHIVE_DB_PORT} + ports: + - '${ARCHIVE_EXPLORER_PORT}:3000' volumes: chain-data: @@ -436,3 +490,5 @@ volumes: driver: local orion-db-data: driver: local + orion_archive_db_data: + driver: local diff --git a/orion-postgres.conf b/orion-postgres.conf new file mode 100644 index 0000000000..f40a4a261a --- /dev/null +++ b/orion-postgres.conf @@ -0,0 +1,4 @@ +listen_addresses = '*' +log_statement = all +autovacuum_analyze_scale_factor = 0.01 +shared_buffers=1GB diff --git a/start-orion.sh b/start-orion.sh index 9c9325876d..58edc75926 100755 --- a/start-orion.sh +++ b/start-orion.sh @@ -2,7 +2,10 @@ set -e docker-compose up -d orion-db -docker-compose up -d squid-archive -docker-compose up -d squid-archive-gateway docker-compose up -d orion-processor -docker-compose up -d orion-api \ No newline at end of file +docker-compose up -d orion-graphql-api +docker-compose up -d orion-auth-api +docker-compose up -d squid-archive-ingest +docker-compose up -d squid-archive-db +docker-compose up -d squid-archive-gateway +docker-compose up -d squid-archive-explorer