From 0522a072f10bdb567058ad3a6cda75bf4d5d9382 Mon Sep 17 00:00:00 2001 From: Utkir S Date: Thu, 5 Oct 2023 14:45:57 +0500 Subject: [PATCH] feat: Fix rocksdb on arm (#97) * switch to @farcasterxyz/rocksdb * update rocksdb in node package * fix typeroots * fix broken dependency & rebuild bcrypto * chore(release): 1.0.4-alpha --- Dockerfile | 4 +- lerna.json | 2 +- package.json | 7 +- packages/api/package.json | 6 +- packages/cli/package.json | 12 +- packages/db/package.json | 6 +- packages/db/src/rocksDb.ts | 2 +- packages/db/tsconfig.build.json | 9 +- packages/executor/package.json | 6 +- packages/node/package.json | 17 ++- .../node/src/network/peers/datastore-rocks.ts | 2 +- packages/node/tsconfig.build.json | 6 +- packages/params/package.json | 6 +- packages/types/package.json | 2 +- packages/utils/package.json | 4 +- types/rocksdb/index.d.ts | 122 ++++++++++++++++++ yarn.lock | 18 +-- 17 files changed, 184 insertions(+), 47 deletions(-) create mode 100644 types/rocksdb/index.d.ts diff --git a/Dockerfile b/Dockerfile index d3c8f86e..5eec1b75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18-alpine as build_src +FROM --platform=${BUILDPLATFORM:-amd64} node:18-alpine as build_src WORKDIR /usr/app RUN apk update && apk add --no-cache g++ make python3 && rm -rf /var/cache/apk/* @@ -18,6 +18,8 @@ COPY --from=build_src /usr/app . RUN yarn install --non-interactive --frozen-lockfile --production --force RUN npx lerna@6.4.1 bootstrap --ignore-scripts -- --production --no-optional +RUN cd node_modules/bcrypto && yarn install + FROM node:18-alpine WORKDIR /usr/app COPY --from=build_deps /usr/app . diff --git a/lerna.json b/lerna.json index 0f699292..17a9bd77 100644 --- a/lerna.json +++ b/lerna.json @@ -4,7 +4,7 @@ ], "npmClient": "yarn", "useWorkspaces": true, - "version": "1.0.3-alpha", + "version": "1.0.4-alpha", "stream": "true", "command": { "version": { diff --git a/package.json b/package.json index a0ec70aa..27efe5a5 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "root", "private": true, - "version": "1.0.3-alpha", + "version": "1.0.4-alpha", "engines": { "node": ">=18.0.0" }, @@ -17,7 +17,12 @@ "release:publish": "lerna publish from-package --yes --no-verify-access", "release:docker": "exit 0" }, + "dependencies": { + "abstract-leveldown": "7.2.0", + "lru-cache": "^6.0.0" + }, "devDependencies": { + "@types/abstract-leveldown": "7.2.1", "@types/compression": "1.7.2", "@types/jest": "29.2.3", "@types/node": "18.11.9", diff --git a/packages/api/package.json b/packages/api/package.json index 1d7e62b4..d32deeec 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "api", - "version": "1.0.3-alpha", + "version": "1.0.4-alpha", "description": "The API module of Etherspot bundler client", "author": "Etherspot", "homepage": "https://https://github.com/etherspot/skandha#readme", @@ -35,12 +35,12 @@ "class-transformer": "0.5.1", "class-validator": "0.14.0", "ethers": "5.7.2", - "executor": "1.0.3-alpha", + "executor": "1.0.4-alpha", "fastify": "4.14.1", "pino": "8.11.0", "pino-pretty": "10.0.0", "reflect-metadata": "0.1.13", - "types": "1.0.3-alpha" + "types": "1.0.4-alpha" }, "devDependencies": { "@types/connect": "3.4.35" diff --git a/packages/cli/package.json b/packages/cli/package.json index 4fbc78f6..78e25a94 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "cli", - "version": "1.0.3-alpha", + "version": "1.0.4-alpha", "description": "> TODO: description", "author": "zincoshine ", "homepage": "https://https://github.com/etherspot/skandha#readme", @@ -38,14 +38,14 @@ "@libp2p/peer-id-factory": "2.0.1", "@libp2p/prometheus-metrics": "1.1.3", "@multiformats/multiaddr": "12.1.3", - "api": "1.0.3-alpha", - "db": "1.0.3-alpha", - "executor": "1.0.3-alpha", + "api": "1.0.4-alpha", + "db": "1.0.4-alpha", + "executor": "1.0.4-alpha", "find-up": "5.0.0", "got": "12.5.3", "js-yaml": "4.1.0", - "node": "1.0.3-alpha", - "types": "1.0.3-alpha", + "node": "1.0.4-alpha", + "types": "1.0.4-alpha", "yargs": "17.6.2" }, "devDependencies": { diff --git a/packages/db/package.json b/packages/db/package.json index 158e9b91..9ace999a 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "db", - "version": "1.0.3-alpha", + "version": "1.0.4-alpha", "description": "The DB module of Etherspot bundler client", "author": "Etherspot", "homepage": "https://github.com/etherspot/etherspot-bundler#readme", @@ -32,8 +32,8 @@ }, "dependencies": { "@chainsafe/ssz": "0.10.1", - "rocksdb": "5.2.1", - "types": "1.0.3-alpha" + "@farcaster/rocksdb": "5.5.0", + "types": "1.0.4-alpha" }, "devDependencies": { "@types/rocksdb": "3.0.1", diff --git a/packages/db/src/rocksDb.ts b/packages/db/src/rocksDb.ts index 7184da7e..c1161732 100644 --- a/packages/db/src/rocksDb.ts +++ b/packages/db/src/rocksDb.ts @@ -1,5 +1,5 @@ import path from "node:path"; -import rocks from "rocksdb"; +import rocks from "@farcaster/rocksdb"; // eslint-disable-next-line import/no-extraneous-dependencies import { IDbController } from "types/lib"; diff --git a/packages/db/tsconfig.build.json b/packages/db/tsconfig.build.json index 680caeb7..278e83e3 100644 --- a/packages/db/tsconfig.build.json +++ b/packages/db/tsconfig.build.json @@ -3,6 +3,11 @@ "include": ["src"], "compilerOptions": { "outDir": "lib", - "typeRoots": ["../../node_modules/@types", "./node_modules/@types", "../../types"] - } + "typeRoots": [ + "../../types", + "../../node_modules/@types", + "./node_modules/@types", + ] + }, + "noImplicitAny": false, } \ No newline at end of file diff --git a/packages/executor/package.json b/packages/executor/package.json index c3589e30..3d80e780 100644 --- a/packages/executor/package.json +++ b/packages/executor/package.json @@ -1,6 +1,6 @@ { "name": "executor", - "version": "1.0.3-alpha", + "version": "1.0.4-alpha", "description": "The Relayer module of Etherspot bundler client", "author": "Etherspot", "homepage": "https://https://github.com/etherspot/skandha#readme", @@ -33,7 +33,7 @@ "dependencies": { "async-mutex": "0.4.0", "ethers": "5.7.2", - "params": "1.0.3-alpha", - "types": "1.0.3-alpha" + "params": "1.0.4-alpha", + "types": "1.0.4-alpha" } } diff --git a/packages/node/package.json b/packages/node/package.json index d012c95e..ceb4e6f3 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -1,6 +1,6 @@ { "name": "node", - "version": "1.0.3-alpha", + "version": "1.0.4-alpha", "description": "The bundler node module of Etherspot bundler client", "author": "Etherspot", "homepage": "https://https://github.com/etherspot/skandha#readme", @@ -42,6 +42,7 @@ "@chainsafe/ssz": "0.10.2", "@chainsafe/threads": "1.10.0", "@ethersproject/abi": "5.7.0", + "@farcaster/rocksdb": "5.5.0", "@libp2p/bootstrap": "6.0.3", "@libp2p/interface-connection": "3.0.2", "@libp2p/interface-connection-manager": "1.3.8", @@ -55,31 +56,29 @@ "@libp2p/tcp": "6.1.0", "@multiformats/multiaddr": "11.4.0", "abstract-leveldown": "7.2.0", - "api": "1.0.3-alpha", + "api": "1.0.4-alpha", "datastore-core": "8.0.1", - "db": "1.0.3-alpha", + "db": "1.0.4-alpha", "ethers": "5.7.2", - "executor": "1.0.3-alpha", + "executor": "1.0.4-alpha", "it-filter": "1.0.2", "it-map": "1.0.5", "it-sort": "1.0.0", "it-take": "1.0.1", "libp2p": "0.42.2", - "params": "1.0.3-alpha", + "params": "1.0.4-alpha", "prettier": "2.8.4", - "rocksdb": "5.2.1", "snappy": "7.2.2", "snappyjs": "0.7.0", "stream-to-it": "0.2.4", "strict-event-emitter-types": "2.0.0", - "types": "1.0.3-alpha", - "utils": "1.0.3-alpha", + "types": "1.0.4-alpha", + "utils": "1.0.4-alpha", "varint": "6.0.0", "xxhash-wasm": "1.0.2" }, "devDependencies": { "@types/abstract-leveldown": "7.2.1", - "@types/rocksdb": "3.0.1", "@types/varint": "6.0.1" }, "overrides": { diff --git a/packages/node/src/network/peers/datastore-rocks.ts b/packages/node/src/network/peers/datastore-rocks.ts index 93680974..c1392cdd 100644 --- a/packages/node/src/network/peers/datastore-rocks.ts +++ b/packages/node/src/network/peers/datastore-rocks.ts @@ -4,7 +4,7 @@ import filter from "it-filter"; import map from "it-map"; import take from "it-take"; import sort from "it-sort"; -import rocks from "rocksdb"; +import rocks from "@farcaster/rocksdb"; import { AbstractOpenOptions, AbstractIteratorOptions, diff --git a/packages/node/tsconfig.build.json b/packages/node/tsconfig.build.json index 1f800138..fcd2fa7e 100644 --- a/packages/node/tsconfig.build.json +++ b/packages/node/tsconfig.build.json @@ -3,7 +3,11 @@ "include": ["src"], "compilerOptions": { "outDir": "lib", - "typeRoots": ["../../node_modules/@types", "./node_modules/@types", "../../types"] + "typeRoots": [ + "../../types", + "../../node_modules/@types", + "./node_modules/@types" + ] }, "noImplicitAny": false, } \ No newline at end of file diff --git a/packages/params/package.json b/packages/params/package.json index 640d49e2..9f6687f9 100644 --- a/packages/params/package.json +++ b/packages/params/package.json @@ -1,6 +1,6 @@ { "name": "params", - "version": "1.0.3-alpha", + "version": "1.0.4-alpha", "description": "Various bundler parameters", "author": "Etherspot", "homepage": "https://github.com/etherspot/skandha#readme", @@ -26,8 +26,8 @@ "@eth-optimism/sdk": "3.0.0", "@mantleio/sdk": "0.2.1", "ethers": "5.7.2", - "types": "1.0.3-alpha", - "utils": "1.0.3-alpha" + "types": "1.0.4-alpha", + "utils": "1.0.4-alpha" }, "scripts": { "clean": "rm -rf lib && rm -f *.tsbuildinfo", diff --git a/packages/types/package.json b/packages/types/package.json index 61f58311..45fc0089 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "types", - "version": "1.0.3-alpha", + "version": "1.0.4-alpha", "description": "The types of Etherspot bundler client", "author": "Etherspot", "homepage": "https://https://github.com/etherspot/skandha#readme", diff --git a/packages/utils/package.json b/packages/utils/package.json index d7348253..2fd49d6b 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -1,6 +1,6 @@ { "name": "utils", - "version": "1.0.3-alpha", + "version": "1.0.4-alpha", "description": "utils of Etherspot bundler client", "author": "Etherspot", "homepage": "https://https://github.com/etherspot/skandha#readme", @@ -37,6 +37,6 @@ "case": "^1.6.3", "pino": "8.11.0", "pino-pretty": "10.0.0", - "types": "1.0.3-alpha" + "types": "1.0.4-alpha" } } diff --git a/types/rocksdb/index.d.ts b/types/rocksdb/index.d.ts new file mode 100644 index 00000000..5ee86e16 --- /dev/null +++ b/types/rocksdb/index.d.ts @@ -0,0 +1,122 @@ +// Type definitions for @farcaster/rocksdb 3.0 +// Project: https://github.com/farcasterxyz/rocksdb +// Definitions by: Meirion Hughes +// Daniel Byrne +// Paul Fletcher-Hill +// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped +// TypeScript Version: 2.3 + +/// + +// If we ever start using the upstream rocksdb package again, remove this +// wrapper `module` declaration. It's only needed because our forked package is +// scoped to @farcaster. +declare module "@farcaster/rocksdb" { + import { + AbstractBatch, + AbstractChainedBatch, + AbstractGetOptions, + AbstractIterator, + AbstractIteratorOptions, + AbstractLevelDOWN, + AbstractOpenOptions, + AbstractOptions, + ErrorCallback, + ErrorValueCallback, + } from "abstract-leveldown"; + + interface RocksDB extends AbstractLevelDOWN { + open(cb: ErrorCallback): void; + open(options: RocksDB.OpenOptions, cb: ErrorCallback): void; + + get(key: RocksDB.Bytes, cb: ErrorValueCallback): void; + get( + key: RocksDB.Bytes, + options: RocksDB.GetOptions, + cb: ErrorValueCallback + ): void; + + put(key: RocksDB.Bytes, value: RocksDB.Bytes, cb: ErrorCallback): void; + put( + key: RocksDB.Bytes, + value: RocksDB.Bytes, + options: RocksDB.PutOptions, + cb: ErrorCallback + ): void; + + del(key: RocksDB.Bytes, cb: ErrorCallback): void; + del( + key: RocksDB.Bytes, + options: RocksDB.DelOptions, + cb: ErrorCallback + ): void; + + batch(): AbstractChainedBatch; + batch( + array: AbstractBatch[], + cb: ErrorCallback + ): AbstractChainedBatch; + batch( + array: AbstractBatch[], + options: RocksDB.BatchOptions, + cb: ErrorCallback + ): AbstractChainedBatch; + + approximateSize( + start: RocksDB.Bytes, + end: RocksDB.Bytes, + cb: RocksDB.ErrorSizeCallback + ): void; + compactRange( + start: RocksDB.Bytes, + end: RocksDB.Bytes, + cb: ErrorCallback + ): void; + getProperty(property: string): string; + iterator(options?: RocksDB.IteratorOptions): RocksDB.Iterator; + } + + namespace RocksDB { + type Bytes = string | Buffer; + type ErrorSizeCallback = (err: Error | undefined, size: number) => void; + type OpenOptions = AbstractOpenOptions; + + interface GetOptions extends AbstractGetOptions { + fillCache?: boolean | undefined; + } + + interface PutOptions extends AbstractOptions { + sync?: boolean | undefined; + } + + interface DelOptions extends AbstractOptions { + sync?: boolean | undefined; + } + + interface BatchOptions extends AbstractOptions { + sync?: boolean | undefined; + } + + interface IteratorOptions extends AbstractIteratorOptions { + fillCache?: boolean | undefined; + } + + interface Iterator extends AbstractIterator { + seek(key: Bytes): void; + binding: any; + cache: any; + finished: any; + fastFuture: any; + } + + interface Constructor { + new (location: string): RocksDB; + (location: string): RocksDB; + destroy(location: string, cb: ErrorCallback): void; + repair(location: string, cb: ErrorCallback): void; + } + } + + const RocksDB: RocksDB.Constructor; + export default RocksDB; +} diff --git a/yarn.lock b/yarn.lock index 0d264572..27fd2f09 100644 --- a/yarn.lock +++ b/yarn.lock @@ -979,6 +979,15 @@ "@ethersproject/properties" "^5.7.0" "@ethersproject/strings" "^5.7.0" +"@farcaster/rocksdb@5.5.0": + version "5.5.0" + resolved "https://registry.yarnpkg.com/@farcaster/rocksdb/-/rocksdb-5.5.0.tgz#c352480a7d6e08c2eee8e1690dc0a49e5a72c8fd" + integrity sha512-E5xavqAOrS9yXV2zw2lEyv7j4w1jZd0UNtsi5C9tqk5Kw9voCojA/zTgb5WugEqHAT6Bt/szahhnNuWxbo9NPg== + dependencies: + abstract-leveldown "^7.2.0" + napi-macros "^2.0.0" + node-gyp-build "^4.3.0" + "@fastify/ajv-compiler@^3.5.0": version "3.5.0" resolved "https://registry.yarnpkg.com/@fastify/ajv-compiler/-/ajv-compiler-3.5.0.tgz#459bff00fefbf86c96ec30e62e933d2379e46670" @@ -9854,15 +9863,6 @@ rlp@^2.2.6, rlp@^2.2.7: dependencies: bn.js "^5.2.0" -rocksdb@5.2.1: - version "5.2.1" - resolved "https://registry.yarnpkg.com/rocksdb/-/rocksdb-5.2.1.tgz#40fd54b798f8843290521313462f4347b702e60d" - integrity sha512-SN9RRLfqDEGfZ6MgfjpQ4DIn3+TdN5ECKYHgeuPKFy6Yw0I9hYyE46giLidngjZU47JXUmWXJDZuU1CyEcf4Fw== - dependencies: - abstract-leveldown "^7.2.0" - napi-macros "^2.0.0" - node-gyp-build "^4.3.0" - run-async@^2.4.0: version "2.4.1" resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.4.1.tgz#8440eccf99ea3e70bd409d49aab88e10c189a455"