rename public keys env

.env.example

@@ -1,13 +1,13 @@
-# ClickHouse Database
+# ClickHouse DB (optional)
 HOST=http://127.0.0.1:8123
 USERNAME=default
 DATABASE=default
 PASSWORD=
 
-# Webhook Authentication (Optional)
-PUBLIC_KEY=... # ed25519 public key provided by https://github.com/pinax-network/substreams-sink-webhook
+# Webhook Ed25519 signature (Optional)
+PUBLIC_KEYS=... # ed25519 public key provided by https://github.com/pinax-network/substreams-sink-webhook
 
-# Clickhouse Sink (Optional)
+# Sink HTTP server (Optional)
 PORT=3000
 HOSTNAME=0.0.0.0
 VERBOSE=true

README.md

@@ -6,11 +6,9 @@
 
 - [Executable binaries](#executable-binaries)
 - [Features](#features)
-- [First steps](#first-steps)
+- [Quickstart](#quickstart)
+- [`.env`](#environment)
 - [CLI](#cli)
-- [Environment](#environment)
-- [Database structure](#database-structure)
-- [Development](#development)
 
 ## Executable [binaries](https://github.com/pinax-network/substreams-sink-clickhouse/releases)
 
@@ -28,114 +26,90 @@ See detailed [list of features](/docs/features.md).
   - Materialized View
 - Automatic block metadata
 - Serveless
-- Authentication
-  - Basic authentication for users
-  - ed25519 signatures for [webhooks](https://github.com/pinax-network/substreams-sink-webhook)
+- [Webhooks](https://github.com/pinax-network/substreams-sink-webhook) Ed25519 signatures
 - No data loss
 
-## First steps
+## Quickstart
 
 Every request can also be executed via the [online UI](http://localhost:3000).
 
-1. Start the sink
+1. Associate Webhook Ed25519 public keys from [substreams-sink-webhook](https://github.com/pinax-network/substreams-sink-webhook).
 
    ```bash
-   $ ./substreams-sink-clickhouse
+   $ echo "PUBLIC_KEYS=<PK1>,<PK2>,..." >> .env
    ```
 
-1. Protect the sink with a password
-
-   ```bash
-   $ AUTH_KEY=$(curl --location 'localhost:3000/hash' --header 'Content-Type: text/plain' --data '<password>')
-   $ echo "AUTH_KEY=$AUTH_KEY" > .env
-   ```
-
-1. Associate public keys from [substreams-sink-webhook](https://github.com/pinax-network/substreams-sink-webhook)s
+1. Start the sink
 
    ```bash
-   $ echo "PUBLIC_KEY=<PK1>,<PK2>,..." >> .env
+   $ ./substreams-sink-clickhouse
    ```
 
 1. Initialize the database (_set database credentials in [environment](#environment)_)
 
    ```bash
    $ ./substream-sink-clickhouse
    $ curl --location 'localhost:3000/health' # --> OK
-   $ curl --location --request PUT "localhost:3000/init" --header "Authorization: Bearer <password>" # --> OK
+   $ curl --location --request PUT "localhost:3000/init" # --> OK
    ```
 
-1. Create a table for your substreams (more details on [table initialization](/docs/features.md#table-initialization))
+1. Create TABLE for your EntityChanges Substreams (more details on [table initialization](/docs/features.md#table-initialization))
 
    ```bash
    $ curl --location --request PUT "localhost:3000/schema/sql" \
        --header "Content-Type: text/plain" \
-       --header "Authorization: Bearer <password>" \
        --data "CREATE TABLE foo () ENGINE=MergeTree ORDER BY();"
    $ # OR
-   $ curl --location --request PUT 'localhost:3000/schema/sql?schema-url=<url>' --header 'Authorization: Bearer <password>'
+   $ curl --location --request PUT 'localhost:3000/schema/sql?schema-url=<url>'
    ```
 
-## CLI
-
-Each field in [environment](#environment) can be overriden when starting the sink.
-
-```bash
-$ ./substreams-sink-clickhouse --help
-```
-
-```
-Substreams Clickhouse sink module
-
-Options:
-  -V, --version                      output the version number
-  -p, --port <number>                HTTP port on which to attach the sink (default: "3000", env: PORT)
-  -v, --verbose <boolean>            Enable verbose logging (choices: "true", "false", default: "true", env: VERBOSE)
-  --hostname <string>                Server listen on HTTP hostname (default: "0.0.0.0", env: HOSTNAME)
-  --public-keys <string>             Comma separated list of public keys to validate messages (env: PUBLIC_KEYS)
-  --host <string>                    Database HTTP hostname (default: "http://localhost:8123", env: HOST)
-  --username <string>                Database user (default: "default", env: USERNAME)
-  --password <string>                Password associated with the specified username (default: "", env: PASSWORD)
-  --database <string>                The database to use inside ClickHouse (default: "default", env: DATABASE)
-  --allow-unparsed <boolean>         Enable storage in 'unparsed_json' table (choices: "true", "false", default: false,
-                                     env: ALLOW_UNPARSED)
-  -h, --help                         display help for command
-```
-
 ## Environment
 
 ```bash
 $ cp .env.example .env
 ```
 
 ```bash
-# ClickHouse Database
+# ClickHouse DB (optional)
 HOST=http://127.0.0.1:8123
 USERNAME=default
 DATABASE=default
 PASSWORD=
 
-# Webhook Authentication (Optional)
-PUBLIC_KEY=... # ed25519 public key provided by https://github.com/pinax-network/substreams-sink-webhook
+# Webhook Ed25519 signature (Optional)
+PUBLIC_KEYS=... # ed25519 public key provided by https://github.com/pinax-network/substreams-sink-webhook
 
-# HTTP Server (Optional)
+# Sink HTTP server (Optional)
 PORT=3000
 HOSTNAME=0.0.0.0
-
-# Clickhouse Sink Authentication (Optional)
-# PUT endpoints are protected (uses HTTP Basic authentication)
-AUTH_KEY=...
-
-# Clickhouse Sink (Optional)
-MAX_BUFFER_SIZE=1000
-INSERTION_DELAY=2000
-WAIT_FOR_INSERT=0
-ASYNC_INSERT=1
-BUFFER=buffer.db
-ALLOW_UNPARSED=false
 VERBOSE=true
-RESUME=true
 ```
 
+## CLI
+
+Each field in [environment](#environment) can be overriden when starting the sink.
+
+```bash
+$ ./substreams-sink-clickhouse --help
+```
+
+```
+Substreams Clickhouse Sink
+
+Options:
+  -V, --version            output the version number
+  -v, --verbose <boolean>  Enable verbose logging (choices: "true", "false", default: "true", env: VERBOSE)
+  -p, --port <number>      Sink HTTP server port (default: "3000", env: PORT)
+  --hostname <string>      Sink HTTP server hostname (default: "0.0.0.0", env: HOSTNAME)
+  --public-keys <string>   Webhook Ed25519 public keys (comma separated) (env: PUBLIC_KEYS)
+  --host <string>          Clickhouse DB hostname (default: "http://localhost:8123", env: HOST)
+  --username <string>      Clickhouse DB username (default: "default", env: USERNAME)
+  --password <string>      Clickhouse DB password (default: "", env: PASSWORD)
+  --database <string>      Clickhouse DB database (default: "default", env: DATABASE)
+  -h, --help               display help for command
+```
+
+
 ## Database structure
 
 See [detailed documentation](/docs/database.md)

index.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-import { config } from "./src/config.js";
+import { config, publicKeys } from "./src/config.js";
 import { name, version } from "./package.json" assert { type: "json" };
 import DELETE from "./src/fetch/DELETE.js";
 import GET from "./src/fetch/GET.js";
@@ -29,8 +29,10 @@ const app = Bun.serve({
 });
 
 logger.info('[app]\t', `${name} v${version}`);
-logger.info('[app]\t', `Server listening on http://${app.hostname}:${app.port}`);
-logger.info('[app]\t', `Clickhouse Server ${config.host} (${config.database})`);
-if (config.publicKey) logger.info('[app]\t', `Webhook Ed25519 Public Key: ${config.publicKey}`);
+logger.info('[app]\t', `Sink Server listening on http://${app.hostname}:${app.port}`);
+logger.info('[app]\t', `Clickhouse DB ${config.host} (${config.database})`);
+for ( const publicKey of publicKeys ) {
+  logger.info('[app]\t', `Webhook Ed25519 public key (${publicKey})`);
+}
 await init();
 await show_tables();

package.json

@@ -1,7 +1,7 @@
 {
   "name": "substreams-sink-clickhouse",
   "version": "0.3.0",
-  "description": "Substreams Clickhouse sink module",
+  "description": "Substreams Clickhouse Sink",
   "type": "module",
   "homepage": "https://github.com/pinax-network/substreams-sink-clickhouse",
   "license": "MIT",

sql/tables/tables.ts

@@ -1,14 +1,7 @@
-import { join, parse } from "path";
-import * as glob from "glob";
-import { logger } from "../../src/logger.js";
+import blocks from "./blocks.sql";
+import module_hashes from "./module_hashes.sql";
 
-const pattern = join(import.meta.dirname, "*.sql");
-
-export async function getTables() {
-  const tables: [string, string][] = [];
-  for (const path of glob.sync(pattern)) {
-    tables.push([parse(path).name, await Bun.file(path).text()])
-  }
-  logger.info("[sql::tables]\t", `Loading ${tables.length} tables from ${pattern}`);
-  return tables;
-}
+export const tables = [
+  ["blocks", await Bun.file(blocks).text()],
+  ["module_hashes", await Bun.file(module_hashes).text()]
+];

src/clickhouse/stores.ts

@@ -8,7 +8,8 @@ export let paused = false;
 
 export function pause(value: boolean) {
   paused = value;
-  logger.info('[store::pause]', `\tPaused=${paused}`);
+  logger.info('[store::pause]', `\tpaused=${paused}`);
+  return value;
 }
 
 export async function query_chains() {

src/clickhouse/table-initialization.ts

@@ -1,11 +1,11 @@
 import { logger } from "../logger.js";
 import { client } from "./createClient.js";
 import { augmentCreateTableStatement, getTableName, isCreateTableStatement } from "./table-utils.js";
-import { getTables } from "../../sql/tables/tables.js";
+import { tables } from "../../sql/tables/tables.js";
 
 export async function initializeDefaultTables() {
   const results = [];
-  for ( const [ table, query ] of await getTables() ) {
+  for ( const [ table, query ] of tables ) {
     logger.info('[clickhouse::initializeDefaultTables]\t', `CREATE TABLE [${table}]`);
     results.push({table, query, ...await client.exec({ query })});
   }

src/config.ts

@@ -19,16 +19,24 @@ export const opts = program
   .version(version)
   .description(description)
   .showHelpAfterError()
-  .addOption(new Option("-p, --port <number>", "HTTP port on which to attach the sink").env("PORT").default(DEFAULT_PORT))
   .addOption(new Option("-v, --verbose <boolean>", "Enable verbose logging").choices(["true", "false"]).env("VERBOSE").default(DEFAULT_VERBOSE))
-  .addOption(new Option("--hostname <string>", "Server listen on HTTP hostname").env("HOSTNAME").default(DEFAULT_HOSTNAME))
-  .addOption(new Option("--public-key <string>", "Comma separated list of public keys to validate messages").env("PUBLIC_KEY"))
-  .addOption(new Option("--host <string>", "Database HTTP hostname").env("HOST").default(DEFAULT_HOST))
-  .addOption(new Option("--username <string>", "Database user").env("USERNAME").default(DEFAULT_USERNAME))
-  .addOption(new Option("--password <string>", "Password associated with the specified username").env("PASSWORD").default(DEFAULT_PASSWORD))
-  .addOption(new Option("--database <string>", "The database to use inside ClickHouse").env("DATABASE").default(DEFAULT_DATABASE))
+  .addOption(new Option("-p, --port <number>", "Sink HTTP server port").env("PORT").default(DEFAULT_PORT))
+  .addOption(new Option("--hostname <string>", "Sink HTTP server hostname").env("HOSTNAME").default(DEFAULT_HOSTNAME))
+  .addOption(new Option("--public-key <string>", "Webhook Ed25519 public keys (comma separated)").env("PUBLIC_KEY").hideHelp())
+  .addOption(new Option("--public-keys <string>", "Webhook Ed25519 public keys (comma separated)").env("PUBLIC_KEYS"))
+  .addOption(new Option("--host <string>", "Clickhouse DB hostname").env("HOST").default(DEFAULT_HOST))
+  .addOption(new Option("--username <string>", "Clickhouse DB username").env("USERNAME").default(DEFAULT_USERNAME))
+  .addOption(new Option("--password <string>", "Clickhouse DB password").env("PASSWORD").default(DEFAULT_PASSWORD))
+  .addOption(new Option("--database <string>", "Clickhouse DB database").env("DATABASE").default(DEFAULT_DATABASE))
   .parse()
   .opts();
 
 // Validate Commander argument & .env options
 export const config = ConfigSchema.parse(opts);
+
+// validate public key
+export const publicKeys: string[] = [];
+for ( const publicKey of [...config.publicKeys ?? [], ...config.publicKey ?? []] ) {
+  if ( publicKey.length !== 64 ) throw new Error("Invalid Ed25519 public key length");
+  publicKeys.push(publicKey);
+}

src/fetch/POST.ts

@@ -1,6 +1,6 @@
 import { handleSinkRequest } from "../clickhouse/handleSinkRequest.js";
 import * as store from "../clickhouse/stores.js";
-import { config } from "../config.js";
+import { publicKeys } from "../config.js";
 import { logger } from "../logger.js";
 import * as prometheus from "../prometheus.js";
 import { BodySchema } from "../schemas.js";
@@ -9,13 +9,16 @@ import { toText } from "./cors.js";
 
 export default async function (req: Request) {
   if (store.paused) {
-    return toText("sink is paused", 400);
+    return toText("sink is paused", 500);
   }
 
-  // validate Ed25519 signature
+  // POST body messagefrom Webhook
   const text = await req.text();
-  if ( config.publicKey ) {
-    const signatureResult = await signatureEd25519(req, text);
+
+  // validate Ed25519 signature
+  // if no public keys are set, skip signature verification
+  if ( publicKeys.length ) {
+    const signatureResult = await signatureEd25519(req, text, publicKeys);
     if (!signatureResult.success) return signatureResult.error;
   }
 

src/fetch/PUT.ts

@@ -1,6 +1,6 @@
+import { pause } from "../clickhouse/stores.js";
 import { NotFound, toText } from "./cors.js";
 import init from "./init.js";
-import { handlePause } from "./pause.js";
 import { handleSchemaRequest } from "./schema.js";
 
 export default async function (req: Request): Promise<Response> {
@@ -10,8 +10,8 @@ export default async function (req: Request): Promise<Response> {
     if (pathname === "/init") return await init();
     if (pathname === "/schema/sql") return handleSchemaRequest(req, "sql");
     if (pathname === "/schema/graphql") return handleSchemaRequest(req, "graphql");
-    if (pathname === "/pause") return handlePause(true);
-    if (pathname === "/unpause") return handlePause(false);
+    if (pathname === "/pause") return toText(String(pause(true)));
+    if (pathname === "/unpause") return toText(String(pause(false)));
   } catch (e) {
     console.error(e);
     return toText(String(e), 500);

src/schemas.ts

@@ -1,20 +1,16 @@
+import z from "zod";
 import { DatabaseChanges } from "@substreams/sink-database-changes/zod";
 import { EntityChanges } from "@substreams/sink-entity-changes/zod";
 import { makeBodySchema, makePayloadBody } from "substreams-sink-webhook/schemas";
-import z from "zod";
 
-export const boolean = z
-  .string()
-  .transform((str) => str.toLowerCase() === "true")
-  .or(z.boolean());
+export const boolean = z.string().transform((str) => str.toLowerCase() === "true").or(z.boolean());
 export const positiveNumber = z.coerce.number().pipe(z.number().positive());
 export const oneOrZero = z.coerce.number().pipe(z.literal(0).or(z.literal(1)));
+export const splitString = (separator: string ) => z.optional(z.string().transform((str) => str.split(separator)));
 
 export const ConfigSchema = z.object({
-  publicKey: z.optional(
-    z.string()
-    .transform((str) => str.split(","))
-    .refine((keys) => keys.filter((key) => key.length > 0).length > 0, "No primary key has been set")),
+  publicKey: splitString(","),
+  publicKeys: splitString(","),
   port: positiveNumber,
   verbose: boolean,
   host: z.string(),