Colossus: Rework sync and cleanup

.gitignore

@@ -48,3 +48,5 @@ runtime-inputs/
 devops/infrastructure
 
 joystream.tar.gz
+
+.venv

colossus.Dockerfile

@@ -30,7 +30,10 @@ RUN yarn workspace storage-node build
 RUN yarn cache clean
 
 FROM node:18 as final
+
 WORKDIR /joystream
+# 7zip and zstd are required by the archive script
+RUN apt-get update && apt-get install -y p7zip-full zstd
 COPY --from=builder /joystream /joystream
 RUN yarn --frozen-lockfile --production
 

setup.sh

@@ -8,7 +8,7 @@ if [[ "$OSTYPE" == "linux-gnu" ]]; then
     # code build tools
     sudo apt-get update -y
     sudo apt-get install -y coreutils clang llvm jq curl gcc xz-utils sudo pkg-config \
-      unzip libc6-dev make libssl-dev python3 cmake protobuf-compiler libprotobuf-dev
+      unzip libc6-dev make libssl-dev python3 cmake protobuf-compiler libprotobuf-dev p7zip-full
 
     # Docker: do not replace existing installation to avoid distrupting running containers
     if ! command -v docker &> /dev/null
@@ -23,7 +23,7 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then
     fi
     # install additional packages
     brew update
-    brew install coreutils gnu-tar jq curl llvm gnu-sed cmake protobuf || :
+    brew install coreutils gnu-tar jq curl llvm gnu-sed cmake protobuf p7zip || :
     echo "It is recommended to setup Docker desktop from: https://www.docker.com/products/docker-desktop"
     echo "It is also recommended to install qemu emulators with following command:"
     echo "docker run --privileged --rm tonistiigi/binfmt --install all"

storage-node/CHANGELOG.md

@@ -1,3 +1,14 @@
+### 4.3.0
+
+- **New feature:** `archive` mode / command, which allows downloading, compressing and uploading all data objects to an external S3 bucket that can be used as a backup.
+- **Optimizations:** The way data objects / data object ids are queried and processed during sync and cleanup has been optimized:
+    - `DataObjectDetailsLoader` and `DataObjectIdsLoader` were implemented. They allow loading data objects / data object ids in batches using a connection query and avoid fetching redundant data from the GraphQL server.
+    - Sync and cleanup services now process tasks in batches of `10_000` to avoid overflowing the memory.
+    - Synchronous operations like `sort` or `filter` on larger arrays of data objects have been optimized (for example, by replacing `.filter(Array.includes(...))` with `.filter(Set.has(...))`).
+- A safety mechanism was added to avoid removing "deleted" objects for which a related `DataObjectDeleted` event cannot be found in storage squid.
+- Improved logging during cleanup.
+
+
 ### 4.2.0
 
 - Fix `util:cleanup` script (call `loadDataObjectIdCache` first)

storage-node/package.json

@@ -1,14 +1,16 @@
 {
   "name": "storage-node",
   "description": "Joystream storage subsystem.",
-  "version": "4.2.0",
+  "version": "4.3.0",
   "author": "Joystream contributors",
   "bin": {
     "storage-node": "./bin/run"
   },
   "bugs": "https://github.com/Joystream/joystream/issues",
   "dependencies": {
     "@apollo/client": "^3.3.21",
+    "@aws-sdk/client-s3": "^3.675.0",
+    "@aws-sdk/s3-request-presigner": "^3.675.0",
     "@elastic/ecs-winston-format": "^1.3.1",
     "@joystream/metadata-protobuf": "^2.15.0",
     "@joystream/opentelemetry": "1.0.0",
@@ -36,6 +38,7 @@
     "await-lock": "^2.1.0",
     "base64url": "^3.0.1",
     "blake3-wasm": "^2.1.5",
+    "chokidar": "4.0.1",
     "cors": "^2.8.5",
     "cross-fetch": "^3.1.4",
     "express": "4.17.1",
@@ -52,6 +55,7 @@
     "node-cache": "^5.1.2",
     "openapi-editor": "^0.3.0",
     "promise-timeout": "^1.3.0",
+    "proper-lockfile": "^4.1.2",
     "react": "^18.2.0",
     "read-chunk": "^3.2.0",
     "rimraf": "^3.0.2",
@@ -81,6 +85,7 @@
     "@types/mocha": "^5",
     "@types/node": "^18.6.0",
     "@types/pg": "^8.6.1",
+    "@types/proper-lockfile": "^4.1.4",
     "@types/swagger-ui-express": "^4.1.2",
     "@types/ws": "^5.1.2",
     "@typescript-eslint/eslint-plugin": "3.8.0",