feat: add script migrations

packages/worker/.env.example

@@ -8,6 +8,10 @@ DATABASE_NAME=block-explorer
 DATABASE_CONNECTION_IDLE_TIMEOUT_MS=12000
 DATABASE_CONNECTION_POOL_SIZE=100
 
+# Used to execute expensive script migrations on background to migrate data without downtime.
+# Script migrations are specific to hosted instances and have to be executed at certain timeframes and controlled manually.
+ENABLE_SCRIPT_MIGRATIONS=false
+
 BLOCKCHAIN_RPC_URL=http://localhost:3050
 DATA_FETCHER_URL=http://localhost:3040
 DATA_FETCHER_REQUEST_TIMEOUT=120000

packages/worker/package.json

@@ -29,7 +29,7 @@
     "migration:create": "npm run typeorm migration:create ./src/migrations/$npm_config_name",
     "migration:run": "npm run typeorm migration:run -- -d ./src/typeorm.config.ts",
     "migration:revert": "npm run typeorm migration:revert -- -d ./src/typeorm.config.ts",
-    "migration-script:run": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register ./src/migrationScripts/{migrationFileName}.ts --runInBand"
+    "script-migrations:run": "ts-node ./scripts/run-script-migrations.ts"
   },
   "dependencies": {
     "@nestjs/axios": "^3.0.0",
@@ -109,7 +109,7 @@
       "src/logger.ts",
       "src/typeorm.config.ts",
       "src/migrations",
-      "src/migrationScripts"
+      "src/scriptMigrations"
     ],
     "reporters": [
       "default",

packages/worker/scripts/run-script-migrations.ts

@@ -0,0 +1,9 @@
+import { Logger } from "@nestjs/common";
+import typeOrmCliDataSource from "../src/typeorm.config";
+import runScriptMigrations from "../src/utils/runScriptMigrations";
+
+(async () => {
+  const logger = new Logger("ScriptMigrations");
+  await runScriptMigrations(typeOrmCliDataSource, logger);
+  logger.log("DONE");
+})();

packages/worker/src/app.module.ts

@@ -31,6 +31,7 @@ import {
   AddressTransferRepository,
   LogRepository,
   BalanceRepository,
+  ScriptMigrationRepository,
 } from "./repositories";
 import {
   Batch,
@@ -44,6 +45,7 @@ import {
   Transfer,
   AddressTransfer,
   Balance,
+  ScriptMigration,
 } from "./entities";
 import { typeOrmModuleOptions } from "./typeorm.config";
 import { JsonRpcProviderModule } from "./rpcProvider/jsonRpcProvider.module";
@@ -80,6 +82,7 @@ import { DataFetcherService } from "./dataFetcher/dataFetcher.service";
       AddressTransfer,
       Transfer,
       Balance,
+      ScriptMigration,
     ]),
     EventEmitterModule.forRoot(),
     JsonRpcProviderModule.forRoot(),
@@ -121,6 +124,7 @@ import { DataFetcherService } from "./dataFetcher/dataFetcher.service";
     AddressTransferRepository,
     BalanceRepository,
     LogRepository,
+    ScriptMigrationRepository,
     BlocksRevertService,
     BatchService,
     BlockProcessor,

packages/worker/src/app.service.ts

@@ -10,6 +10,7 @@ import { CounterService } from "./counter";
 import { BalancesCleanerService } from "./balance";
 import { TokenOffChainDataSaverService } from "./token/tokenOffChainData/tokenOffChainDataSaver.service";
 import runMigrations from "./utils/runMigrations";
+import runScriptMigrations from "./utils/runScriptMigrations";
 
 @Injectable()
 export class AppService implements OnModuleInit, OnModuleDestroy {
@@ -30,6 +31,11 @@ export class AppService implements OnModuleInit, OnModuleDestroy {
 
   public onModuleInit() {
     runMigrations(this.dataSource, this.logger).then(() => {
+      const enableScriptMigrations = this.configService.get<boolean>("scriptMigrations.enabled");
+      if (enableScriptMigrations) {
+        // Run script migrations on background if there are any to run.
+        runScriptMigrations(this.dataSource, this.logger);
+      }
       this.startWorkers();
     });
   }

packages/worker/src/config.spec.ts

@@ -59,6 +59,9 @@ describe("config", () => {
         collectDbConnectionPoolMetricsInterval: 10000,
         collectBlocksToProcessMetricInterval: 10000,
       },
+      scriptMigrations: {
+        enabled: false,
+      },
     });
   });
 });

packages/worker/src/config.ts

@@ -21,6 +21,7 @@ export default () => {
     DISABLE_COUNTERS_PROCESSING,
     DISABLE_OLD_BALANCES_CLEANER,
     DISABLE_BLOCKS_REVERT,
+    ENABLE_SCRIPT_MIGRATIONS,
     ENABLE_TOKEN_OFFCHAIN_DATA_SAVER,
     UPDATE_TOKEN_OFFCHAIN_DATA_INTERVAL,
     SELECTED_TOKEN_OFFCHAIN_DATA_PROVIDER,
@@ -78,5 +79,8 @@ export default () => {
       collectDbConnectionPoolMetricsInterval: parseInt(COLLECT_DB_CONNECTION_POOL_METRICS_INTERVAL, 10) || 10000,
       collectBlocksToProcessMetricInterval: parseInt(COLLECT_BLOCKS_TO_PROCESS_METRIC_INTERVAL, 10) || 10000,
     },
+    scriptMigrations: {
+      enabled: ENABLE_SCRIPT_MIGRATIONS === "true",
+    },
   };
 };

packages/worker/src/entities/index.ts

@@ -13,3 +13,4 @@ export * from "./addressTransfer.entity";
 export * from "./balance.entity";
 export * from "./counter.entity";
 export * from "./counterState.entity";
+export * from "./scriptMigration.entity";

packages/worker/src/entities/scriptMigration.entity.ts

@@ -0,0 +1,35 @@
+import { Entity, Column, PrimaryColumn, Index } from "typeorm";
+import { BaseEntity } from "./base.entity";
+
+export interface ScriptMigrationParams {
+  [key: string]: string;
+}
+
+export enum ScriptMigrationStatus {
+  NotStarted = "not_started",
+  Pending = "pending",
+  Failed = "failed",
+  Completed = "completed",
+  Outdated = "outdated",
+}
+
+@Entity({ name: "scriptMigrations" })
+export class ScriptMigration extends BaseEntity {
+  @PrimaryColumn({ generated: true, type: "bigint" })
+  public readonly number: number;
+
+  @Index({ unique: true })
+  @Column({ type: "varchar" })
+  public readonly name: string;
+
+  @Index()
+  @Column({ type: "bigint" })
+  public readonly timestamp: number;
+
+  @Index()
+  @Column({ type: "enum", enum: ScriptMigrationStatus, default: ScriptMigrationStatus.NotStarted })
+  public readonly status: ScriptMigrationStatus;
+
+  @Column({ type: "jsonb", nullable: true })
+  public readonly params?: ScriptMigrationParams;
+}

packages/worker/src/migrations/1709722093204-AddAddressTransferType.ts

@@ -13,6 +13,12 @@ export class AddAddressTransferType1709722093204 implements MigrationInterface {
     await queryRunner.query(
       `CREATE INDEX "IDX_aa5a147f1f6a4acde1a13de594" ON "addressTransfers" ("address", "type", "timestamp", "logIndex" DESC) `
     );
+    await queryRunner.query(
+      `UPDATE "addressTransfers" Set "type" = "transfers".type::VARCHAR::"addressTransfers_type_enum"
+        FROM "transfers"
+        WHERE "transfers"."number" = "addressTransfers"."transferNumber"
+        AND "transfers"."type" != 'transfer'`
+    );
   }
 
   public async down(queryRunner: QueryRunner): Promise<void> {

packages/worker/src/migrations/1710057320666-AddScriptMigrations.ts

@@ -0,0 +1,25 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddScriptMigrations1710057320666 implements MigrationInterface {
+  name = "AddScriptMigrations1710057320666";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TYPE "public"."scriptMigrations_status_enum" AS ENUM('not_started', 'pending', 'failed', 'completed', 'outdated')`
+    );
+    await queryRunner.query(
+      `CREATE TABLE "scriptMigrations" ("createdAt" TIMESTAMP NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), "number" BIGSERIAL NOT NULL, "name" character varying NOT NULL, "timestamp" bigint NOT NULL, "status" "public"."scriptMigrations_status_enum" NOT NULL DEFAULT 'not_started', "params" jsonb, CONSTRAINT "PK_def0d458be005c5e9640f69dde8" PRIMARY KEY ("number"))`
+    );
+    await queryRunner.query(`CREATE UNIQUE INDEX "IDX_42f873351064e516bd8dcbec67" ON "scriptMigrations" ("name") `);
+    await queryRunner.query(`CREATE INDEX "IDX_7dabb5b44d259202dabf3d1621" ON "scriptMigrations" ("timestamp") `);
+    await queryRunner.query(`CREATE INDEX "IDX_f13cd15275d590b0af3e466907" ON "scriptMigrations" ("status") `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP INDEX "public"."IDX_f13cd15275d590b0af3e466907"`);
+    await queryRunner.query(`DROP INDEX "public"."IDX_7dabb5b44d259202dabf3d1621"`);
+    await queryRunner.query(`DROP INDEX "public"."IDX_42f873351064e516bd8dcbec67"`);
+    await queryRunner.query(`DROP TABLE "scriptMigrations"`);
+    await queryRunner.query(`DROP TYPE "public"."scriptMigrations_status_enum"`);
+  }
+}

packages/worker/src/repositories/index.ts

@@ -11,3 +11,4 @@ export * from "./log.repository";
 export * from "./balance.repository";
 export * from "./counter.repository";
 export * from "./counterState.repository";
+export * from "./scriptMigration.repository";

packages/worker/src/repositories/scriptMigration.repository.ts

@@ -0,0 +1,11 @@
+import { Injectable } from "@nestjs/common";
+import { ScriptMigration } from "../entities";
+import { UnitOfWork } from "../unitOfWork";
+import { BaseRepository } from "./base.repository";
+
+@Injectable()
+export class ScriptMigrationRepository extends BaseRepository<ScriptMigration> {
+  public constructor(unitOfWork: UnitOfWork) {
+    super(ScriptMigration, unitOfWork);
+  }
+}