Use @chainsafe/xdelta3-node package

packages/beacon-node/package.json

@@ -106,6 +106,7 @@
     "@chainsafe/ssz": "^0.17.1",
     "@chainsafe/threads": "^1.11.1",
     "@chainsafe/pubkey-index-map": "2.0.0",
+    "@chainsafe/xdelta3-node": "^1.0.2",
     "@ethersproject/abi": "^5.7.0",
     "@fastify/bearer-auth": "^10.0.1",
     "@fastify/cors": "^10.0.1",
@@ -150,8 +151,6 @@
     "strict-event-emitter-types": "^2.0.0",
     "systeminformation": "^5.22.9",
     "uint8arraylist": "^2.4.7",
-    "xdelta3-wasm": "^1.0.0",
-    "vcdiff-wasm": "^1.0.10",
     "xxhash-wasm": "1.0.2"
   },
   "devDependencies": {

packages/beacon-node/src/chain/historicalState/historicalState.ts

@@ -8,10 +8,10 @@ import {IBeaconDb} from "../../db/interface.js";
 import {HistoricalStateRegenMetrics, IBinaryDiffCodec, StateArchiveStrategy} from "./types.js";
 import {replayBlocks} from "./utils/blockReplay.js";
 import {DiffLayers} from "./diffLayers.js";
-import {BinaryDiffVCDiffCodec} from "./utils/binaryDiffVCDiffCodec.js";
+import {BinaryDiffXDelta3Codec} from "./utils/binaryDiffXDelta3Codec.js";
 import {getDiffState} from "./utils/diff.js";
 
-export const codec: IBinaryDiffCodec = new BinaryDiffVCDiffCodec();
+export const codec: IBinaryDiffCodec = new BinaryDiffXDelta3Codec();
 
 export async function getHistoricalState(
   {slot}: {slot: Slot},

packages/beacon-node/src/chain/historicalState/utils/binaryDiffXDelta3Codec.ts

@@ -1,11 +1,10 @@
-import {init, xd3_encode_memory, xd3_decode_memory, xd3_smatch_cfg} from "xdelta3-wasm";
+import {encodeSync, decodeSync} from "@chainsafe/xdelta3-node";
 import {IBinaryDiffCodec} from "../types.js";
 
 export class BinaryDiffXDelta3Codec implements IBinaryDiffCodec {
   private isInitialized: boolean = false;
 
   async init(): Promise<void> {
-    await init();
     this.isInitialized = true;
   }
 
@@ -14,23 +13,18 @@ export class BinaryDiffXDelta3Codec implements IBinaryDiffCodec {
   }
 
   compute(base: Uint8Array, changed: Uint8Array): Uint8Array {
-    // The max size of a diff can be if input is empty and source is full state
-    // TODO: Try to optimize a way to calculate max output size to reduce memory consumption
-    const delta = xd3_encode_memory(changed, base, 1024 * 1024, xd3_smatch_cfg.FAST);
-    if (delta.str === "SUCCESS") {
-      return delta.output;
+    try {
+      return encodeSync(base, changed);
+    } catch (err) {
+      throw new Error(`Can not compute binary diff error=${(err as Error).message}`);
     }
-
-    throw new Error(`Can not compute binary diff error=${delta.str}`);
   }
 
   apply(base: Uint8Array, delta: Uint8Array): Uint8Array {
-    const orig = xd3_decode_memory(delta, base, 9999);
-
-    if (orig.str === "SUCCESS") {
-      return orig.output;
+    try {
+      return decodeSync(base, delta);
+    } catch (err) {
+      throw new Error(`Can not apply binary diff patch error=${(err as Error).message}`);
     }
-
-    throw new Error("Can not apply binary diff patch");
   }
 }

packages/beacon-node/test/unit/chain/historicalState/utils/binaryDiffCodec.test.ts → packages/beacon-node/test/unit/chain/historicalState/utils/binaryDiffXDelta3Codec.test.ts

@@ -4,7 +4,7 @@ import {describe, it, expect, beforeAll} from "vitest";
 import {BeaconState, Epoch, phase0, RootHex, Slot, ssz} from "@lodestar/types";
 import {fromHex} from "@lodestar/utils";
 import {ForkName} from "@lodestar/params";
-import {BinaryDiffVCDiffCodec} from "../../../../../src/chain/historicalState/utils/binaryDiffVCDiffCodec.js";
+import {BinaryDiffXDelta3Codec} from "../../../../../src/chain/historicalState/utils/binaryDiffXDelta3Codec.js";
 import {generateState} from "../../../../utils/state.js";
 import {IBinaryDiffCodec} from "../../../../../src/chain/historicalState/types.js";
 
@@ -16,8 +16,8 @@ const testsCases: {title: string; base: () => Uint8Array; changed: () => Uint8Ar
   },
   {
     title: "Array of numbers",
-    base: () => Uint8Array.from([10, 11, 12]),
-    changed: () => Uint8Array.from([10, 11, 12, 14, 15]),
+    base: () => Uint8Array.from([10, 11, 12, 13, 14, 15]),
+    changed: () => Uint8Array.from([10, 11, 12, 14, 15, 16, 17, 18]),
   },
   {
     title: "An attestation",
@@ -69,7 +69,7 @@ describe("BinaryDiffCodec", () => {
   let multiDiffData: Record<string, {value: Uint8Array; diff: Uint8Array}>;
 
   beforeAll(async () => {
-    codec = new BinaryDiffVCDiffCodec();
+    codec = new BinaryDiffXDelta3Codec();
     await codec.init();
 
     multiDiffData = {

packages/beacon-node/test/unit/chain/historicalState/utils/diff.test.ts

@@ -6,7 +6,7 @@ import {getMockedBeaconDb} from "../../../../mocks/mockedBeaconDb.js";
 import {getMockedLogger} from "../../../../mocks/loggerMock.js";
 import {getDiffState} from "../../../../../src/chain/historicalState/utils/diff.js";
 import {IBinaryDiffCodec} from "../../../../../src/chain/historicalState/types.js";
-import {BinaryDiffVCDiffCodec} from "../../../../../src/chain/historicalState/utils/binaryDiffVCDiffCodec.js";
+import {BinaryDiffXDelta3Codec} from "../../../../../src/chain/historicalState/utils/binaryDiffXDelta3Codec.js";
 
 describe("historicalState/util", () => {
   let db: IBeaconDb;
@@ -18,7 +18,7 @@ describe("historicalState/util", () => {
     db = getMockedBeaconDb();
     logger = getMockedLogger();
     diffLayers = new DiffLayers();
-    codec = new BinaryDiffVCDiffCodec();
+    codec = new BinaryDiffXDelta3Codec();
 
     vi.spyOn(codec, "apply");
     vi.spyOn(codec, "compute");

yarn.lock

@@ -650,6 +650,54 @@
   optionalDependencies:
     tiny-worker ">= 2"
 
+"@chainsafe/[email protected]":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node-darwin-arm64/-/xdelta3-node-darwin-arm64-1.0.2.tgz#fa1891f29e7e44d0e4149f4b8f5731b333345db1"
+  integrity sha512-sgfhUZg87b/7JpBWLIPKv2tXzamLZNzEZnRDyCIR48r/VLZbslHSNz8m4qNWcsbflt81X9G4QxTz/gqdpSNxHA==
+
+"@chainsafe/[email protected]":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node-darwin-x64/-/xdelta3-node-darwin-x64-1.0.2.tgz#bda8f2eed2544ffa90cb06e90f767b96f8f4ea64"
+  integrity sha512-mOwZTgMqujBTl7gkYX+yoVL4eiM1NvS8cvb9fpLv4kvc9e9T0yMYP1QieDHuLEc2xk27ddZYAzYIXO9c8iDbDQ==
+
+"@chainsafe/[email protected]":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node-linux-arm64-gnu/-/xdelta3-node-linux-arm64-gnu-1.0.2.tgz#0c27c53eb110a27323a59a93d143453c9dc5f445"
+  integrity sha512-/K5oZoMmf1WvcKQAL4lozU7kVxQ+NFtjBOsWQNm61weHRkiv4hBx7Fo7fQ71LIESu7Kyvs44zogrViR9bHn31A==
+
+"@chainsafe/[email protected]":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node-linux-arm64-musl/-/xdelta3-node-linux-arm64-musl-1.0.2.tgz#c2bb58531a8b57c00f8929952b391dd445b10cc7"
+  integrity sha512-GuAgRIUXZFGmAUz/bDAOEuFIexU4EcObIuoqijJWWGmdJYuLQsFkyEtr3YpANQJ0RT6EKYgK6AVHczzuG917AQ==
+
+"@chainsafe/[email protected]":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node-linux-x64-gnu/-/xdelta3-node-linux-x64-gnu-1.0.2.tgz#570184846e0a0ab4216458b73ee54aae3f26bc53"
+  integrity sha512-sL/UjocpJHLI1W+rqUxaOmlP4+nUUjQQCJuaReV1nDErgeQuUfPhYU8wQYRbwA+uCVR2eP2D64V5yehpFyNtcA==
+
+"@chainsafe/[email protected]":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node-linux-x64-musl/-/xdelta3-node-linux-x64-musl-1.0.2.tgz#8d2dffdc44b1b733a6e67a80a54cc9eb73007d35"
+  integrity sha512-iwl17R2731jCVNfNJFLd3dXw7g4B0wR4AQYf/hUtG6xfxcF3fF75zuTfDbgXq3rH6x0vJv7Bn5Vtm+7w1PQtWg==
+
+"@chainsafe/[email protected]":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node-win32-x64-msvc/-/xdelta3-node-win32-x64-msvc-1.0.2.tgz#f86cc56b0c18dbeb713ecc7b29032a0f5b4d1e4e"
+  integrity sha512-aRsLH7Pxq+hVvhCXwYWmlvtTa07f1btKSejK64yOlptIVrL0PEqlI0ocIYFyHPd1oDsfgBW6645/572ASlYOTw==
+
+"@chainsafe/xdelta3-node@^1.0.2":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@chainsafe/xdelta3-node/-/xdelta3-node-1.0.2.tgz#e0892e1e8a6f1ef72a4dac1c76abdecfead56277"
+  integrity sha512-zVuznkGTJ2tq1BJktRtc54zyPBZIHKoAf0cQWyoXs0dCju8PimKS5T8I7uwSRV36OkejxCIzFzYOqAlrnNnKGg==
+  optionalDependencies:
+    "@chainsafe/xdelta3-node-darwin-arm64" "1.0.2"
+    "@chainsafe/xdelta3-node-darwin-x64" "1.0.2"
+    "@chainsafe/xdelta3-node-linux-arm64-gnu" "1.0.2"
+    "@chainsafe/xdelta3-node-linux-arm64-musl" "1.0.2"
+    "@chainsafe/xdelta3-node-linux-x64-gnu" "1.0.2"
+    "@chainsafe/xdelta3-node-linux-x64-musl" "1.0.2"
+    "@chainsafe/xdelta3-node-win32-x64-msvc" "1.0.2"
+
 "@colors/[email protected]":
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9"
@@ -13152,11 +13200,6 @@ validator@^13.7.0:
   resolved "https://registry.yarnpkg.com/validator/-/validator-13.11.0.tgz#23ab3fd59290c61248364eabf4067f04955fbb1b"
   integrity sha512-Ii+sehpSfZy+At5nPdnyMhx78fEoPDkR2XW/zimHEL3MyGJQOCQ7WeP20jPYRz7ZCpcKLB21NxuXHF3bxjStBQ==
 
-vcdiff-wasm@^1.0.10:
-  version "1.0.10"
-  resolved "https://registry.yarnpkg.com/vcdiff-wasm/-/vcdiff-wasm-1.0.10.tgz#e58c53f39fcfd04c5facac6854081b1269fd6080"
-  integrity sha512-07siWwxXYClqAihEtoDYE18f5Z/lsX1ZQnJTVQ3k193bzjKCyOgja7jJBNzzr7jeKBGyF1Byc5CQf8tjPbUHKQ==
-
 [email protected]:
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/vite-node/-/vite-node-2.0.4.tgz#5600cc9f0d9c3ff9a64050c6858e7e1b62fb3fcd"
@@ -13845,11 +13888,6 @@ ws@^8.18.0:
   resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc"
   integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==
 
-xdelta3-wasm@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/xdelta3-wasm/-/xdelta3-wasm-1.0.0.tgz#43b6ef9e9649830e25713914e40324217f8565fd"
-  integrity sha512-vhS28BhVaE3S/PGG1KQIwjBVqJecuS5Sdh82UAZysbnYaU93KS6l8ZPtsqonNZMeuyFgrGW9D44hh2lwQCsidA==
-
 xml-name-validator@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-5.0.0.tgz#82be9b957f7afdacf961e5980f1bf227c0bf7673"