diff --git a/index.ts b/index.ts index 86fef6e..f98f11b 100644 --- a/index.ts +++ b/index.ts @@ -10,13 +10,17 @@ import { parseClock } from "./src/parseClock.js"; import { parseSchema } from "./src/parseSchema.js"; export async function action(options: CSVRunOptions ) { - // @substreams/manifest issue - // if manifest is local, add current directory - if (!isRemotePath(options.manifest) && !path.isAbsolute(options.manifest)) { - const currentDir = process.cwd(); - options.manifest = path.join(currentDir, options.manifest); + // handle file system manifest + // can be removed when issue resolved + // https://github.com/substreams-js/substreams-js/issues/62 + if (!isRemotePath(options.manifest)) { + // if manifest is not absolute, add current directory + if ( !path.isAbsolute(options.manifest)) { + const currentDir = process.cwd(); + options.manifest = path.join(currentDir, options.manifest); + } + if ( !fs.existsSync(options.manifest) ) throw new Error(`Manifest file not found: ${options.manifest}`); } - if ( !fs.existsSync(options.manifest) ) throw new Error(`Manifest file not found: ${options.manifest}`); // SQL schema if ( !fs.existsSync(options.schema) ) throw new Error(`Schema file not found: ${options.schema}`); diff --git a/package.json b/package.json index a5d628d..a6caefe 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "version": "0.2.5", + "version": "0.2.6", "name": "substreams-sink-csv", "description": "Substreams Sink CSV", "type": "module", diff --git a/src/getModuleHash.spec.ts b/src/getModuleHash.spec.ts new file mode 100644 index 0000000..42778ee --- /dev/null +++ b/src/getModuleHash.spec.ts @@ -0,0 +1,8 @@ +import { expect, test } from "bun:test"; +import { isRemotePath } from "./getModuleHash.js"; + +test("isRemotePath", () => { + expect(isRemotePath("https://github.com/streamingfast/substreams-eth-block-meta/releases/download/v0.5.1/substreams-eth-block-meta-v0.5.1.spkg")).toBe(true); + expect(isRemotePath("substreams-eth-block-meta-v0.5.1.spkg")).toBe(false); + expect(isRemotePath("./substreams-eth-block-meta-v0.5.1.spkg")).toBe(false); +}) diff --git a/src/parseSchema.spec.ts b/src/parseSchema.spec.ts index 3ebf393..3fe63ad 100644 --- a/src/parseSchema.spec.ts +++ b/src/parseSchema.spec.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test"; -import { parseColumn, parseCreateTable, parseSchema } from "./parseSchema.js"; +import { parseColumn, parseCreateTable, parseSchema, preParseStatement } from "./parseSchema.js"; test("parseCreateTable", () => { expect(parseCreateTable("CREATE TABLE block_meta")).toBe("block_meta"); @@ -23,6 +23,12 @@ test("parseColumn", () => { expect(parseColumn("CONSTRAINT PK_Person PRIMARY KEY (ID,LastName)")).toBe(""); }) +test("preParseStatement", () => { + expect(preParseStatement("parent_hash TEXT, ")).toBe("parent_hash TEXT"); + expect(preParseStatement(" \"timestamp\" INTEGER ")).toBe("timestamp INTEGER"); + expect(preParseStatement("'timestamp' INTEGER")).toBe("timestamp INTEGER"); +}) + test("parseSchema::factory_pair_created", () => { const sql = ` CREATE TABLE factory_pair_created ( @@ -63,3 +69,86 @@ test("parseSchema::block_meta", () => { const tables = parseSchema(sql); expect(tables).toEqual(new Map([["block_meta", ["id", "at", "number", "hash", "parent_hash", "timestamp"]]])); }); + +test("parseSchema::transfers", () => { + const sql = ` + -- Table for transfers -- + CREATE TABLE IF NOT EXISTS transfers ( + -- trace information + trx_id String, + action_index UInt32, + -- contract & scope -- + contract FixedString(12), + action String, + symcode String, + -- data payload -- + from FixedString(12), + to FixedString(12), + quantity String, + memo String, + -- extras -- + precision UInt32, + amount Int64, + value Float64, + ) + ENGINE = ReplacingMergeTree() + -- primary key = trx_id + action_index -- + PRIMARY KEY (id) + ORDER BY (id); + + -- Table for accounts -- + CREATE TABLE IF NOT EXISTS accounts ( + -- trace information -- + trx_id String, + action_index UInt32, + + -- contract & scope -- + contract FixedString(12), + symcode String, + + -- data payload -- + account FixedString(12), + balance String, + balance_delta Int64, + + -- extras -- + precision UInt32, + amount Int64, + value Float64, + ) + ENGINE = ReplacingMergeTree() + -- primary key = trx_id + action_index -- + PRIMARY KEY (id) + ORDER BY (id); +` + const tables = parseSchema(sql); + expect(tables).toEqual(new Map([[ + "transfers", + [ + "trx_id", + "action_index", + "contract", + "action", + "symcode", + "from", + "to", + "quantity", + "memo", + "precision", + "amount", + "value", + ]],[ + "accounts", + [ + "trx_id", + "action_index", + "contract", + "symcode", + "account", + "balance", + "balance_delta", + "precision", + "amount", + "value", + ]]])); +}); diff --git a/src/parseSchema.ts b/src/parseSchema.ts index 259c3b6..3bcddea 100644 --- a/src/parseSchema.ts +++ b/src/parseSchema.ts @@ -2,18 +2,19 @@ export function parseSchema(sql: string) { const tables = new Map(); // const statements = sql.split(";") - // should return `block_meta` as table and `id, at, number, hash, parent_hash, timestamp` as columns - for (const statement of statements) { + for (let statement of statements) { + statement = preParseStatement(statement); const lines = statement.trim().split("\n"); - const table = parseCreateTable(lines[0]); - // console.log(table, lines); - if ( !table ) continue; const columns = new Set(); + let table = ''; for ( const line of lines) { + const parsedTable = parseCreateTable(line); + if ( parsedTable ) table = parsedTable; + if ( !table ) continue; const column = parseColumn(line); if (column) columns.add(column); } - tables.set(table, Array.from(columns)); + if ( table ) tables.set(table, Array.from(columns)); } return tables; } @@ -24,6 +25,7 @@ export function parseSchema(sql: string) { // create table block meta // CREATE TABLE IF NOT EXISTS block_meta export function parseCreateTable(statement: string) { + statement = preParseStatement(statement); const match = statement.match(/^CREATE TABLE/i); if (match) { statement = statement.replace("(", "").trim(); @@ -37,16 +39,26 @@ export function parseCreateTable(statement: string) { // parent_hash TEXT, // timestamp INTEGER export function parseColumn(statement: string) { - statement = statement.trim().replace(/[,;]/g, ''); // remove trailing comma or semicolon - statement = statement.replace(/[\"\']/g, ''); // remove quotes + statement = preParseStatement(statement); if ( statement.match(/^CREATE TABLE/i) ) return '' // ignore table name if ( statement.match(/^PRIMARY KEY/i) ) return '' // ignore primary key as valid column if ( statement.match(/^\)/) ) return '' // ignore closing parenthesis if ( statement.match(/^\s*$/) ) return '' // ignore empty lines if ( statement.match(/^CONSTRAINT/i) ) return '' // ignore constraints + if ( statement.match(/^ENGINE/i) ) return '' // ignore engine + if ( statement.match(/^ORDER BY/i) ) return '' // ignore engine + if ( statement.match(/^--/i) ) return '' // ignore comments const words = statement.split(" "); if ( words.length > 1) { return words[0].trim(); } return ''; +} + +// remove trailing comma or semicolon +// remove quotes +export function preParseStatement(statement: string) { + return statement + .replace(/[,;\'\"]/g, '') + .trim() } \ No newline at end of file