colossus s3 api

.env

@@ -186,3 +186,18 @@ SQUID_GQL_PORT=4352
 # Archive gateway host (Should not be set in local development)
 # For running a production storage-squid instance uncomment the following line (to use the subsquid hosted archive)
 # SQUID_ARCHIVE_GATEWAY_URL=${CUSTOM_ARCHIVE_GATEWAY_URL:-https://v2.archive.subsquid.io/network/joystream}
+
+# =====================================================================================
+## Cloud storage provider configuration
+
+# Enable or disable the cloud storage provider
+ENABLE_STORAGE_PROVIDER=true
+
+## Specify the cloud storage provider to use:
+CLOUD_STORAGE_PROVIDER_NAME=aws
+
+# AWS S3 (replace with your own configuration and credentials)
+AWS_ACCESS_KEY_ID=test-key
+AWS_SECRET_ACCESS_KEY=test-secret
+AWS_REGION=eu-west-1
+AWS_BUCKET_NAME=sample-bucket

.gitignore

@@ -48,3 +48,6 @@ runtime-inputs/
 devops/infrastructure
 
 joystream.tar.gz
+
+# localstack for s3 backend colossus testing
+localstack/localstack_init

cli/src/Api.ts

@@ -89,7 +89,7 @@
     return this._api as unknown as UnaugmentedApiPromise
   }
 
   private static async initApi(apiUri: string = DEFAULT_API_URI, metadataCache: Record<string, any>) {
     const wsProvider: WsProvider = new WsProvider(apiUri)
     const api = new ApiPromise({ provider: wsProvider, metadata: metadataCache })
     await api.isReadyOrError
@@ -110,7 +110,7 @@
 
   static async create(
     apiUri = DEFAULT_API_URI,
     metadataCache: Record<string, any>,
     qnApi?: QueryNodeApi
   ): Promise<Api> {
     const { api, chainType } = await Api.initApi(apiUri, metadataCache)
@@ -182,7 +182,7 @@
     return new Date(blockTime.toNumber())
   }
 
-  protected workingGroupApiQuery<T extends WorkingGroups>(group: T): ApiPromise['query'][typeof apiModuleByGroup[T]] {
+  protected workingGroupApiQuery<T extends WorkingGroups>(group: T): ApiPromise['query'][(typeof apiModuleByGroup)[T]] {
     const module = apiModuleByGroup[group]
     return this._api.query[module]
   }

cli/src/base/ContentDirectoryCommandBase.ts

@@ -23,10 +23,10 @@ const CATEGORIES_CONTEXTS = ['Lead', 'Curator'] as const
 const MODERATION_ACTION_CONTEXTS = ['Lead', 'Curator'] as const
 const CHANNEL_MANAGEMENT_CONTEXTS = ['Owner', 'Curator', 'Collaborator'] as const
 
-type ChannelManagementContext = typeof CHANNEL_MANAGEMENT_CONTEXTS[number]
-type ChannelCreationContext = typeof CHANNEL_CREATION_CONTEXTS[number]
-type CategoriesContext = typeof CATEGORIES_CONTEXTS[number]
-type ModerationActionContext = typeof MODERATION_ACTION_CONTEXTS[number]
+type ChannelManagementContext = (typeof CHANNEL_MANAGEMENT_CONTEXTS)[number]
+type ChannelCreationContext = (typeof CHANNEL_CREATION_CONTEXTS)[number]
+type CategoriesContext = (typeof CATEGORIES_CONTEXTS)[number]
+type ModerationActionContext = (typeof MODERATION_ACTION_CONTEXTS)[number]
 
 /**
  * Abstract base class for commands related to content directory

cli/src/base/ForumCommandBase.ts

@@ -19,7 +19,7 @@ import {
 import { AccountId } from '@polkadot/types/interfaces'
 const FORUM_MODERATION_CONTEXT = ['Leader', 'Moderator'] as const
 
-type ForumModerationContext = typeof FORUM_MODERATION_CONTEXT[number]
+type ForumModerationContext = (typeof FORUM_MODERATION_CONTEXT)[number]
 
 /**
  * Abstract base class for commands related to forum management

cli/src/commands/api/inspect.ts

@@ -22,7 +22,7 @@ const TYPES_AVAILABLE = ['query', 'consts'] as const
 
 // String literals type based on TYPES_AVAILABLE const.
 // It works as if we specified: type ApiType = 'query' | 'consts'...;
-type ApiType = typeof TYPES_AVAILABLE[number]
+type ApiType = (typeof TYPES_AVAILABLE)[number]
 
 export default class ApiInspect extends ApiCommandBase {
   static description =

cli/src/helpers/display.ts

@@ -49,7 +49,7 @@ export function displayTable(rows: { [k: string]: string | number }[], cellHoriz
     }, columnName.length)
   const columnDef = (columnName: string) => ({
     header: columnName,
-    get: (row: typeof rows[number]) => chalk.magentaBright(`${row[columnName]}`),
+    get: (row: (typeof rows)[number]) => chalk.magentaBright(`${row[columnName]}`),
     minWidth: maxLength(columnName) + cellHorizontalPadding,
   })
   const columns: Table.table.Columns<{ [k: string]: string }> = {}

docker-compose.localstack.yml

@@ -0,0 +1,26 @@
+version: '3.4'
+services:
+  localstack:
+    container_name: local_stack
+    image: localstack/localstack
+    network_mode: bridge
+    environment:
+      - SERVICES=s3 # we only need s3 bucket
+      - DEBUG=1
+      - DEFAULT_REGION=${AWS_REGION}
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - LOCALSTACK_ENDPOINT=${LOCALSTACK_ENDPOINT}
+    ports:
+      - 4566:4566
+    volumes:
+      - localstack-data:/var/lib/localstack'
+
+networks:
+  joystream:
+    external: true
+    name: joystream_default
+
+volumes:
+  localstack-data:
+    driver: local

docker-compose.yml

@@ -9,8 +9,8 @@ services:
       - chain-data:/data
     environment:
       - CHAIN=${CHAIN}
-    command: "--chain ${CHAIN:-dev} --alice --validator --pruning=archive --unsafe-ws-external --unsafe-rpc-external
-      --rpc-methods Safe --rpc-cors=all --log runtime --base-path /data --no-hardware-benchmarks"
+    command: '--chain ${CHAIN:-dev} --alice --validator --pruning=archive --unsafe-ws-external --unsafe-rpc-external
+      --rpc-methods Safe --rpc-cors=all --log runtime --base-path /data --no-hardware-benchmarks'
     ports:
       - 9944:9944
       - 9933:9933
@@ -38,15 +38,20 @@ services:
       - OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT}
       - OTEL_RESOURCE_ATTRIBUTES=service.name=colossus-1,deployment.environment=production
     entrypoint: ['/joystream/entrypoints/storage.sh']
-    command: [
-      'server', '--worker=${COLOSSUS_1_WORKER_ID}', '--port=3333', '--uploads=/data/uploads/',
-      '--sync', '--syncInterval=1',
-      '--storageSquidEndpoint=${COLOSSUS_STORAGE_SQUID_URL}',
-      '--apiUrl=${JOYSTREAM_NODE_WS}',
-      '--logFilePath=/logs',
-      '--tempFolder=/data/temp/',
-      '--pendingFolder=/data/pending/'
-    ]
+    command:
+      [
+        'server',
+        '--worker=${COLOSSUS_1_WORKER_ID}',
+        '--port=3333',
+        '--uploads=/data/uploads/',
+        '--sync',
+        '--syncInterval=1',
+        '--storageSquidEndpoint=${COLOSSUS_STORAGE_SQUID_URL}',
+        '--apiUrl=${JOYSTREAM_NODE_WS}',
+        '--logFilePath=/logs',
+        '--tempFolder=/data/temp/',
+        '--pendingFolder=/data/pending/',
+      ]
 
   distributor-1:
     image: node:18
@@ -71,7 +76,7 @@ services:
     environment:
       JOYSTREAM_DISTRIBUTOR__ID: distributor-1
       JOYSTREAM_DISTRIBUTOR__ENDPOINTS__STORAGE_SQUID: ${DISTRIBUTOR_STORAGE_SQUID_URL}
-      JOYSTREAM_DISTRIBUTOR__KEYS: "[{\"suri\":\"${DISTRIBUTOR_1_ACCOUNT_URI}\"}]"
+      JOYSTREAM_DISTRIBUTOR__KEYS: '[{"suri":"${DISTRIBUTOR_1_ACCOUNT_URI}"}]'
       JOYSTREAM_DISTRIBUTOR__WORKER_ID: ${DISTRIBUTOR_1_WORKER_ID}
       JOYSTREAM_DISTRIBUTOR__PUBLIC_API__PORT: 3334
       JOYSTREAM_DISTRIBUTOR__OPERATOR_API__PORT: 4334
@@ -110,15 +115,20 @@ services:
       # ACCOUNT_URI overrides command line arg --accountUri
       - ACCOUNT_URI=${COLOSSUS_2_TRANSACTOR_URI}
     entrypoint: ['yarn', 'storage-node']
-    command: [
-      'server', '--worker=${COLOSSUS_2_WORKER_ID}', '--port=3333', '--uploads=/data/uploads',
-      '--sync', '--syncInterval=1',
-      '--storageSquidEndpoint=${COLOSSUS_STORAGE_SQUID_URL}',
-      '--apiUrl=${JOYSTREAM_NODE_WS}',
-      '--logFilePath=/logs',
-      '--tempFolder=/data/temp/',
-      '--pendingFolder=/data/pending/'
-    ]
+    command:
+      [
+        'server',
+        '--worker=${COLOSSUS_2_WORKER_ID}',
+        '--port=3333',
+        '--uploads=/data/uploads',
+        '--sync',
+        '--syncInterval=1',
+        '--storageSquidEndpoint=${COLOSSUS_STORAGE_SQUID_URL}',
+        '--apiUrl=${JOYSTREAM_NODE_WS}',
+        '--logFilePath=/logs',
+        '--tempFolder=/data/temp/',
+        '--pendingFolder=/data/pending/',
+      ]
 
   distributor-2:
     image: node:18
@@ -143,7 +153,7 @@ services:
     environment:
       JOYSTREAM_DISTRIBUTOR__ID: distributor-2
       JOYSTREAM_DISTRIBUTOR__ENDPOINTS__STORAGE_SQUID: ${DISTRIBUTOR_STORAGE_SQUID_URL}
-      JOYSTREAM_DISTRIBUTOR__KEYS: "[{\"suri\":\"${DISTRIBUTOR_2_ACCOUNT_URI}\"}]"
+      JOYSTREAM_DISTRIBUTOR__KEYS: '[{"suri":"${DISTRIBUTOR_2_ACCOUNT_URI}"}]'
       JOYSTREAM_DISTRIBUTOR__WORKER_ID: ${DISTRIBUTOR_2_WORKER_ID}
       JOYSTREAM_DISTRIBUTOR__PUBLIC_API__PORT: 3334
       JOYSTREAM_DISTRIBUTOR__OPERATOR_API__PORT: 4334
@@ -193,8 +203,8 @@ services:
       - OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT}
       - OTEL_RESOURCE_ATTRIBUTES=service.name=query-node,deployment.environment=production
     ports:
-      - "${GRAPHQL_SERVER_PORT}:${GRAPHQL_SERVER_PORT}"
-      - "127.0.0.1:${PROCESSOR_STATE_APP_PORT}:${PROCESSOR_STATE_APP_PORT}"
+      - '${GRAPHQL_SERVER_PORT}:${GRAPHQL_SERVER_PORT}'
+      - '127.0.0.1:${PROCESSOR_STATE_APP_PORT}:${PROCESSOR_STATE_APP_PORT}'
     depends_on:
       - db
     volumes:
@@ -279,7 +289,7 @@ services:
       - PORT=${HYDRA_INDEXER_GATEWAY_PORT}
       - PGSSLMODE=disable
     ports:
-      - "${HYDRA_INDEXER_GATEWAY_PORT}:${HYDRA_INDEXER_GATEWAY_PORT}"
+      - '${HYDRA_INDEXER_GATEWAY_PORT}:${HYDRA_INDEXER_GATEWAY_PORT}'
     depends_on:
       - db
       - redis
@@ -289,7 +299,7 @@ services:
     container_name: redis
     restart: unless-stopped
     ports:
-      - "127.0.0.1:6379:6379"
+      - '127.0.0.1:6379:6379'
 
   faucet:
     image: joystream/faucet:carthage
@@ -308,7 +318,7 @@ services:
       - BALANCE_CREDIT=${BALANCE_CREDIT}
       - BALANCE_LOCKED=${BALANCE_LOCKED}
     ports:
-      - "3002:3002"
+      - '3002:3002'
 
   # PostgerSQL database for Orion
   orion-db:
@@ -441,10 +451,7 @@ services:
     environment:
       DATABASE_MAX_CONNECTIONS: 5
       RUST_LOG: 'actix_web=info,actix_server=info'
-    command: [
-        '--database-url',
-        'postgres://postgres:postgres@orion_archive_db:${ARCHIVE_DB_PORT}/squid-archive',
-      ]
+    command: ['--database-url', 'postgres://postgres:postgres@orion_archive_db:${ARCHIVE_DB_PORT}/squid-archive']
     ports:
       - '127.0.0.1:${ARCHIVE_GATEWAY_PORT}:8000'
       - '[::1]:${ARCHIVE_GATEWAY_PORT}:8000'

storage-node/client/api.ts

@@ -63,6 +63,12 @@ export interface DataStatsResponse {
      * @memberof DataStatsResponse
      */
     'pendingObjects'?: number;
+    /**
+     * 
+     * @type {string}
+     * @memberof DataStatsResponse
+     */
+    'cloudProvider'?: string;
 }
 /**
  * 

storage-node/jest.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  testMatch: ['**/*.test.ts'],
+};

storage-node/package.json

@@ -9,6 +9,8 @@
   "bugs": "https://github.com/Joystream/joystream/issues",
   "dependencies": {
     "@apollo/client": "^3.3.21",
+    "@aws-sdk/client-s3": "^3.577.0",
+    "@aws-sdk/s3-request-presigner": "^3.577.0",
     "@elastic/ecs-winston-format": "^1.3.1",
     "@joystream/metadata-protobuf": "^2.15.0",
     "@joystream/opentelemetry": "1.0.0",
@@ -72,6 +74,7 @@
     "@graphql-codegen/typescript": "^1.22.0",
     "@graphql-codegen/typescript-document-nodes": "^1.17.11",
     "@graphql-codegen/typescript-operations": "^1.17.16",
+    "@jest/globals": "^29.7.0",
     "@joystream/eslint-config": "^1.0.0",
     "@oclif/dev-cli": "^1",
     "@oclif/test": "^1",
@@ -90,6 +93,7 @@
     "eslint-config-oclif": "^3.1",
     "eslint-config-oclif-typescript": "^0.1",
     "globby": "^10",
+    "jest": "^29.7.0",
     "json-schema-to-typescript": "^10.1.4",
     "mocha": "^5",
     "nyc": "^14",
@@ -98,6 +102,7 @@
     "prettier": "^2.3.0",
     "sinon": "^11.1.1",
     "swagger-ui-express": "^4.1.6",
+    "ts-jest": "^29.1.2",
     "ts-node": "^10.2.1",
     "type-doc": "^0.1.41",
     "typescript": "^5.0.2"
@@ -167,6 +172,7 @@
     "ensure": "yarn format && yarn lint --fix && yarn build",
     "checks": "tsc --noEmit --pretty && prettier ./src --check && yarn lint",
     "start": "./bin/run server",
+    "test:integration:cloudProvider": "jest --detectOpenHandles './src/services/storageProviders/tests/**/syncService.test.ts'",
     "start:with-instrumentation": "export OTEL_APPLICATION=storage-node; export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8200; export OTEL_RESOURCE_ATTRIBUTES=service.name=colossus,deployment.environment=production; node --require @joystream/opentelemetry ./bin/run server"
   },
   "types": "lib/index.d.ts"

storage-node/src/api-spec/openapi.yaml

@@ -259,6 +259,8 @@ components:
         pendingObjects:
           type: integer
           format: int64
+        cloudProvider:
+          type: string 
     VersionResponse:
       type: object
       required:

storage-node/src/command-base/ExitCodes.ts

@@ -14,6 +14,7 @@ enum ExitCodes {
   ApiError = 200,
   UnsuccessfulRuntimeCall,
   KeyringNotReady,
+  CacheInitError,
 
   // NOTE: never exceed exit code 255 or it will be modulated by `256` and create problems
 }

storage-node/src/commands/server.ts

@@ -22,8 +22,14 @@ import { getStorageBucketIdsByWorkerId } from '../services/sync/storageObligatio
 import { PendingDirName, TempDirName, performSync } from '../services/sync/synchronizer'
 import { createApp } from '../services/webApi/app'
 import ExitCodes from './../command-base/ExitCodes'
+import { IConnectionHandler, parseConfigOptionAndBuildConnection } from '../services/storageProviders'
 const fsPromises = fs.promises
 
+// Global variable for storage provider connection, initialised by server, then readonly
+let storageProviderConnection: IConnectionHandler | undefined
+export const getStorageProviderConnection = () => storageProviderConnection
+export const isStorageProviderConnectionEnabled = () => storageProviderConnection !== undefined
+
 /**
  * CLI command:
  * Starts the storage node server.
@@ -228,8 +234,8 @@ Supported values: warn, error, debug, info. Default:debug`,
       logger.warn(`Only subset of buckets will process uploads!`)
     }
 
-    logger.info(`Buckets synced and served: ${selectedBuckets}`)
-    logger.info(`Buckets accepting uploads: ${writableBuckets}`)
+    logger.info(`Buckets synced and served: [${selectedBuckets}]`)
+    logger.info(`Buckets accepting uploads: [${writableBuckets}]`)
 
     if (!flags.tempFolder) {
       logger.warn(
@@ -260,6 +266,10 @@ Supported values: warn, error, debug, info. Default:debug`,
       this.error('Paths for pending and uploads folders must be unique.')
     }
 
+    // initialise storage provider connection: undefined if not enabled
+    storageProviderConnection = await parseConfigOptionAndBuildConnection()
+    logger.debug(`remote storage provider connection status: ${isStorageProviderConnectionEnabled()}`)
+
     await createDirectory(flags.uploads)
     await loadDataObjectIdCache(flags.uploads)
 

storage-node/src/commands/util/types.ts

@@ -0,0 +1 @@
+export type Maybe<T> = T | undefined