Support Storage Migration

README.md

@@ -260,7 +260,8 @@ git clone https://github.com/ls1intum/Apollon_standalone.git
 
 #### STEP 2: Configure the environment
 
-> [!NOTE] You can skip this step for local deployment.
+> [!NOTE]
+> You can skip this step for local deployment.
 
 Add a `.env` file in the root folder of the code. Add the following variables:
 

docker-compose.prod.yml

@@ -19,6 +19,7 @@ services:
     environment:
       - APOLLON_REDIS_URL=redis://apollon-redis:6379
       - APOLLON_REDIS_DIAGRAM_TTL=${APOLLON_REDIS_DIAGRAM_TTL}
+      - APOLLON_REDIS_MIGRATE_FROM_FILE=true
       - DEPLOYMENT_URL=${DEPLOYMENT_URL}
     volumes:
       - /opt/apollon/diagrams:/app/diagrams

docker-compose.yml

@@ -13,12 +13,11 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.redis
-      args:
-        - DEPLOYMENT_URL=${DEPLOYMENT_URL}
     container_name: apollon_standalone
     environment:
       - APOLLON_REDIS_URL=redis://apollon_redis:6379
       - APOLLON_REDIS_DIAGRAM_TTL=${APOLLON_REDIS_DIAGRAM_TTL}
+      - APOLLON_REDIS_MIGRATE_FROM_FILE=true
       - DEPLOYMENT_URL=${DEPLOYMENT_URL}
     volumes:
       - ./diagrams:/app/diagrams

packages/server/src/main/services/diagram-storage/index.ts

@@ -3,6 +3,7 @@ export { DiagramStorageService } from './diagram-storage-service';
 import { DiagramStorageService } from './diagram-storage-service';
 import { DiagramRedisStorageService } from './diagram-redis-storage-service';
 import { DiagramFileStorageService } from './diagram-file-storage-service';
+import { MigratingStorageService } from './migrating-storage-service';
 
 /**
  * Factory for creating a diagram storage service. Will determine
@@ -13,10 +14,20 @@ export class DiagramStorageFactory {
 
   private static createStorageService(): DiagramStorageService {
     if (process.env.APOLLON_REDIS_URL !== undefined) {
-      return new DiagramRedisStorageService({
+
+      const redisStorage = new DiagramRedisStorageService({
         url: process.env.APOLLON_REDIS_URL,
         ttl: process.env.APOLLON_REDIS_DIAGRAM_TTL,
       });
+
+      if (process.env.APOLLON_REDIS_MIGRATE_FROM_FILE !== undefined) {
+        return new MigratingStorageService({
+          targetStorage: redisStorage,
+          sourceStorage: new DiagramFileStorageService(),
+        });
+      } else {
+        return redisStorage;
+      }
     } else {
       return new DiagramFileStorageService();
     }

packages/server/src/main/services/diagram-storage/migrating-storage-service.ts

@@ -0,0 +1,64 @@
+import { Operation } from 'fast-json-patch';
+import { DiagramDTO } from 'shared/src/diagram-dto';
+import { DiagramStorageService } from './diagram-storage-service';
+
+
+/**
+ * Options for a migrating storage service.
+ */
+export interface MigratingStorageOptions {
+  /**
+   * The storage service to migrate to.
+   * - New diagrams will be stored in this storage.
+   * - Loading old diagrams will cause them to be saved in this storage.
+   * - Updates to old diagrams are applied to this storage.
+   */
+  targetStorage: DiagramStorageService;
+
+  /**
+   * The storage service to load old diagrams from.
+   * - If a diagram is loaded from this storage and updated,
+   *   the changes WILL NOT be saved in this storage.
+   */
+  sourceStorage: DiagramStorageService;
+}
+
+
+/**
+ * Storage service that migrates diagrams between two storage services.
+ * This class allows for migrating from one diagram storage backend to another
+ * in a slow rollout, and without any downtime, assuming both storage backends
+ * are available during the migration.
+ * 
+ * - New diagrams are stored solely in the target storage.
+ * - Upon loading diagrams from the old storage, they are also stored in target storage.
+ * - Updates are only applied to target storage, so diagrams in the source storage
+ *   will slowly become outdated.
+ */
+export class MigratingStorageService implements DiagramStorageService {
+  constructor(readonly options: MigratingStorageOptions) {}
+
+  async saveDiagram(diagramDTO: DiagramDTO, token: string) {
+    return this.options.targetStorage.saveDiagram(diagramDTO, token);
+  }
+
+  async patchDiagram(token: string, patch: Operation[]) {
+    await this.options.targetStorage.patchDiagram(token, patch);
+  }
+
+  async diagramExists(token: string) {
+    return await this.options.targetStorage.diagramExists(token)
+      || await this.options.sourceStorage.diagramExists(token);
+  }
+
+  async getDiagramByLink(token: string) {
+    if (await this.options.targetStorage.diagramExists(token)) {
+      return await this.options.targetStorage.getDiagramByLink(token);
+    } else {
+      const dto = await this.options.sourceStorage.getDiagramByLink(token);
+      dto && await this.options.targetStorage.saveDiagram(dto, token);
+
+      return dto;
+    }
+  }
+}