Tag Block/Tx backfills as done instead of removing them

0xProject · Jan 29, 2024 · bf145c3 · bf145c3
1 parent a552943
commit bf145c3
Show file tree

Hide file tree

Showing 11 changed files with 194 additions and 4 deletions.
diff --git a/migrations/arbitrum/1706550739000-AddDoneToTxBackfillTable.ts b/migrations/arbitrum/1706550739000-AddDoneToTxBackfillTable.ts
@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner, getConnection } from 'typeorm';
+
+export class AddDoneToTxBackfillTable1706550739000 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`
+          ALTER TABLE ${schema}.tx_backfill ADD CONSTRAINT tx_backfill_pk PRIMARY KEY (transaction_hash, block_number);
+          ALTER TABLE ${schema}.tx_backfill ADD done boolean NOT NULL DEFAULT FALSE;
+          CREATE INDEX tx_backfill_partial ON ${schema}.tx_backfill (transaction_hash, block_number)
+WHERE done = FALSE;
+        `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`ALTER TABLE ${schema}.tx_backfill UPDATE DROP COLUMN done;`);
+    }
+}
diff --git a/migrations/avalanche/1706550739000-AddDoneToTxBackfillTable.ts b/migrations/avalanche/1706550739000-AddDoneToTxBackfillTable.ts
@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner, getConnection } from 'typeorm';
+
+export class AddDoneToTxBackfillTable1706550739000 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`
+          ALTER TABLE ${schema}.tx_backfill ADD CONSTRAINT tx_backfill_pk PRIMARY KEY (transaction_hash, block_number);
+          ALTER TABLE ${schema}.tx_backfill ADD done boolean NOT NULL DEFAULT FALSE;
+          CREATE INDEX tx_backfill_partial ON ${schema}.tx_backfill (transaction_hash, block_number)
+WHERE done = FALSE;
+        `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`ALTER TABLE ${schema}.tx_backfill UPDATE DROP COLUMN done;`);
+    }
+}
diff --git a/migrations/base/1706550739000-AddDoneToTxBackfillTable.ts b/migrations/base/1706550739000-AddDoneToTxBackfillTable.ts
@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner, getConnection } from 'typeorm';
+
+export class AddDoneToTxBackfillTable1706550739000 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`
+          ALTER TABLE ${schema}.tx_backfill ADD CONSTRAINT tx_backfill_pk PRIMARY KEY (transaction_hash, block_number);
+          ALTER TABLE ${schema}.tx_backfill ADD done boolean NOT NULL DEFAULT FALSE;
+          CREATE INDEX tx_backfill_partial ON ${schema}.tx_backfill (transaction_hash, block_number)
+WHERE done = FALSE;
+        `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`ALTER TABLE ${schema}.tx_backfill UPDATE DROP COLUMN done;`);
+    }
+}
diff --git a/migrations/bsc/1706550739000-AddDoneToTxBackfillTable.ts b/migrations/bsc/1706550739000-AddDoneToTxBackfillTable.ts
@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner, getConnection } from 'typeorm';
+
+export class AddDoneToTxBackfillTable1706550739000 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`
+          ALTER TABLE ${schema}.tx_backfill ADD CONSTRAINT tx_backfill_pk PRIMARY KEY (transaction_hash, block_number);
+          ALTER TABLE ${schema}.tx_backfill ADD done boolean NOT NULL DEFAULT FALSE;
+          CREATE INDEX tx_backfill_partial ON ${schema}.tx_backfill (transaction_hash, block_number)
+WHERE done = FALSE;
+        `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`ALTER TABLE ${schema}.tx_backfill UPDATE DROP COLUMN done;`);
+    }
+}
diff --git a/migrations/celo/1706550739000-AddDoneToTxBackfillTable.ts b/migrations/celo/1706550739000-AddDoneToTxBackfillTable.ts
@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner, getConnection } from 'typeorm';
+
+export class AddDoneToTxBackfillTable1706550739000 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`
+          ALTER TABLE ${schema}.tx_backfill ADD CONSTRAINT tx_backfill_pk PRIMARY KEY (transaction_hash, block_number);
+          ALTER TABLE ${schema}.tx_backfill ADD done boolean NOT NULL DEFAULT FALSE;
+          CREATE INDEX tx_backfill_partial ON ${schema}.tx_backfill (transaction_hash, block_number)
+WHERE done = FALSE;
+        `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`ALTER TABLE ${schema}.tx_backfill UPDATE DROP COLUMN done;`);
+    }
+}
diff --git a/migrations/ethereum/1706550739000-AddDoneToTxBackfillTable.ts b/migrations/ethereum/1706550739000-AddDoneToTxBackfillTable.ts
@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner, getConnection } from 'typeorm';
+
+export class AddDoneToTxBackfillTable1706550739000 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`
+          ALTER TABLE ${schema}.tx_backfill ADD CONSTRAINT tx_backfill_pk PRIMARY KEY (transaction_hash, block_number);
+          ALTER TABLE ${schema}.tx_backfill ADD done boolean NOT NULL DEFAULT FALSE;
+          CREATE INDEX tx_backfill_partial ON ${schema}.tx_backfill (transaction_hash, block_number)
+WHERE done = FALSE;
+        `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`ALTER TABLE ${schema}.tx_backfill UPDATE DROP COLUMN done;`);
+    }
+}
diff --git a/migrations/fantom/1706550739000-AddDoneToTxBackfillTable.ts b/migrations/fantom/1706550739000-AddDoneToTxBackfillTable.ts
@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner, getConnection } from 'typeorm';
+
+export class AddDoneToTxBackfillTable1706550739000 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`
+          ALTER TABLE ${schema}.tx_backfill ADD CONSTRAINT tx_backfill_pk PRIMARY KEY (transaction_hash, block_number);
+          ALTER TABLE ${schema}.tx_backfill ADD done boolean NOT NULL DEFAULT FALSE;
+          CREATE INDEX tx_backfill_partial ON ${schema}.tx_backfill (transaction_hash, block_number)
+WHERE done = FALSE;
+        `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`ALTER TABLE ${schema}.tx_backfill UPDATE DROP COLUMN done;`);
+    }
+}
diff --git a/migrations/optimism/1706550739000-AddDoneToTxBackfillTable.ts b/migrations/optimism/1706550739000-AddDoneToTxBackfillTable.ts
@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner, getConnection } from 'typeorm';
+
+export class AddDoneToTxBackfillTable1706550739000 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`
+          ALTER TABLE ${schema}.tx_backfill ADD CONSTRAINT tx_backfill_pk PRIMARY KEY (transaction_hash, block_number);
+          ALTER TABLE ${schema}.tx_backfill ADD done boolean NOT NULL DEFAULT FALSE;
+          CREATE INDEX tx_backfill_partial ON ${schema}.tx_backfill (transaction_hash, block_number)
+WHERE done = FALSE;
+        `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`ALTER TABLE ${schema}.tx_backfill UPDATE DROP COLUMN done;`);
+    }
+}
diff --git a/migrations/polygon/1706550739000-AddDoneToTxBackfillTable.ts b/migrations/polygon/1706550739000-AddDoneToTxBackfillTable.ts
@@ -0,0 +1,20 @@
+import { MigrationInterface, QueryRunner, getConnection } from 'typeorm';
+
+export class AddDoneToTxBackfillTable1706550739000 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`
+          ALTER TABLE ${schema}.tx_backfill ADD CONSTRAINT tx_backfill_pk PRIMARY KEY (transaction_hash, block_number);
+          ALTER TABLE ${schema}.tx_backfill ADD done boolean NOT NULL DEFAULT FALSE;
+          CREATE INDEX tx_backfill_partial ON ${schema}.tx_backfill (transaction_hash, block_number)
+WHERE done = FALSE;
+        `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        const connection = getConnection();
+        const { schema } = connection.options as any;
+        await queryRunner.query(`ALTER TABLE ${schema}.tx_backfill UPDATE DROP COLUMN done;`);
+    }
+}
diff --git a/src/scripts/pull_and_save_backfill_tx.ts b/src/scripts/pull_and_save_backfill_tx.ts
@@ -21,7 +21,11 @@ export class BackfillTxScraper {
         logger.info(`pulling tx backlog`);
 
         const queryResult = await connection.query(
-            `SELECT transaction_hash FROM ${SCHEMA}.tx_backfill LIMIT ${MAX_TX_TO_PULL}`,
+            `SELECT transaction_hash
+             FROM ${SCHEMA}.tx_backfill
+             WHERE done = FALSE
+             ORDER BY block_number
+             LIMIT ${MAX_TX_TO_PULL}`,
         );
 
         const txList = queryResult.map((e: { transaction_hash: string }) => e.transaction_hash);
@@ -34,7 +38,10 @@ export class BackfillTxScraper {
             const txDeleteQuery = `DELETE FROM ${SCHEMA}.transactions WHERE transaction_hash IN (${txHashList})`;
             const txReceiptDeleteQuery = `DELETE FROM ${SCHEMA}.transaction_receipts WHERE transaction_hash IN (${txHashList});`;
             // const txLogsDeleteQuery = `DELETE FROM ${SCHEMA}.transaction_logs WHERE transaction_hash IN (${txHashList});`;
-            const txBacklogQuery = `DELETE FROM ${SCHEMA}.tx_backfill WHERE transaction_hash IN (${txHashList});`;
+            const txBacklogQuery = `
+              UPDATE ${SCHEMA}.tx_backfill
+              SET done = true
+              WHERE block_number IN (${txHashList})`;
 
             const queryRunner = connection.createQueryRunner();
             await queryRunner.connect();

diff --git a/src/scripts/pull_and_save_block_events.ts b/src/scripts/pull_and_save_block_events.ts
@@ -347,6 +347,7 @@ export class BlockEventsScraper {
         const oldestBlocksToBackfill = await connection.query(
             `SELECT DISTINCT block_number
              FROM ${SCHEMA}.tx_backfill
+             WHERE done = false
              ORDER BY block_number
              LIMIT ${MAX_BLOCKS_TO_PULL}`,
         );
@@ -361,11 +362,13 @@ export class BlockEventsScraper {
             const newBlocks = await web3Source.getBatchBlockInfoAsync(blockNumbers, true);
             const success = await getParseSaveBlocksTransactionsEvents(connection, producer, newBlocks, false);
             if (success) {
+                const newBlockNumbers = newBlocks.map((block) => block.number);
                 const queryRunner = connection.createQueryRunner();
                 await queryRunner.connect();
                 await queryRunner.manager.query(
-                    `DELETE FROM ${SCHEMA}.tx_backfill
-                         WHERE block_number IN (${blockNumbers.join(',')})`,
+                    `UPDATE ${SCHEMA}.tx_backfill
+                     SET done = true
+                     WHERE block_number IN (${newBlockNumbers.join(',')})`,
                 );
                 queryRunner.release();