query shore databases

README.md

@@ -14,6 +14,10 @@
 
 See [releases](https://github.com/pinax-network/substreams-sink-clickhouse/releases)
 
+```bash
+$ ./substreams-sink-clickhouse
+```
+
 ## [Features](/docs/features.md)
 
 See detailed [list of features](/docs/features.md).
@@ -39,12 +43,6 @@ Every request can also be executed via the [online UI](http://localhost:3000).
    $ echo "PUBLIC_KEYS=<PK1>,<PK2>,..." >> .env
    ```
 
-1. Start the sink
-
-   ```bash
-   $ ./substreams-sink-clickhouse
-   ```
-
 1. Initialize the database (_set database credentials in [environment](#environment)_)
 
    ```bash
@@ -65,11 +63,13 @@ Every request can also be executed via the [online UI](http://localhost:3000).
 
 ## Environment
 
+Create a `.env` file in the root of the project.
+
 ```bash
 $ cp .env.example .env
 ```
 
-```bash
+```env
 # ClickHouse DB (optional)
 HOST=http://127.0.0.1:8123
 USERNAME=default
@@ -93,7 +93,7 @@ Each field in [environment](#environment) can be overriden when starting the sin
 $ ./substreams-sink-clickhouse --help
 ```
 
-```
+```bash
 Substreams Clickhouse Sink
 
 Options:

index.ts

@@ -10,7 +10,7 @@ import PUT from "./src/fetch/PUT.js";
 import { NotFound } from "./src/fetch/cors.js";
 import { logger } from "./src/logger.js";
 import init from "./src/fetch/init.js";
-import { show_tables } from "./src/clickhouse/stores.js";
+import { show_databases, show_tables } from "./src/clickhouse/stores.js";
 import "./src/exitHandler.js"
 
 if (config.verbose) logger.enable();
@@ -34,5 +34,6 @@ logger.info('[app]\t', `Clickhouse DB ${config.host} (${config.database})`);
 for ( const publicKey of publicKeys ) {
   logger.info('[app]\t', `Webhook Ed25519 public key (${publicKey})`);
 }
-await init();
-await show_tables();
+await show_tables();
+await show_databases();
+await init();

sql/tables/tables.ts

@@ -1,7 +1,7 @@
 import blocks from "./blocks.sql";
 import module_hashes from "./module_hashes.sql";
 
-export const tables = [
+export const sqlTables = [
   ["blocks", await Bun.file(blocks).text()],
   ["module_hashes", await Bun.file(module_hashes).text()]
 ];

src/clickhouse/createDatabase.ts

@@ -1,11 +1,13 @@
 import { logger } from "../logger.js";
 import { client } from "./createClient.js";
+import { databases } from "./stores.js";
 
 export async function createDatabase(database: string) {
   if (!database) {
     throw new Error("[database] is required")
   }
-  logger.info('[clickhouse::createDatabase]\t', `CREATE DATABASE [${database}]`);
+  if ( databases?.has(database) ) return {};
   const query = `CREATE DATABASE IF NOT EXISTS "${database}"`;
+  logger.info('[clickhouse::createDatabase]\t', `CREATE DATABASE [${database}]`);
   return {query, ...await client.exec({ query })};
 }

src/clickhouse/stores.ts

@@ -4,6 +4,7 @@ import { readOnlyClient } from "./createClient.js";
 export let chains: string[] | null = null;
 export let module_hashes: string[] | null = null;
 export let tables: Set<string> | null = null;
+export let databases: Set<string> | null = null;
 export let paused = false;
 
 export function pause(value: boolean) {
@@ -54,3 +55,15 @@ export async function show_tables() {
 
   return tables;
 }
+
+export async function show_databases() {
+  const response = await readOnlyClient.query({
+    query: "SHOW DATABASES",
+    format: "JSONEachRow",
+  });
+  const data = await response.json<{name: string}[]>();
+  databases = new Set(data.map(({ name }) => name));
+  logger.info('[store::show_databases]', `Loaded ${databases.size} databases (${[...databases].join(", ")})`);
+
+  return databases;
+}

src/clickhouse/table-initialization.ts

@@ -1,11 +1,13 @@
 import { logger } from "../logger.js";
 import { client } from "./createClient.js";
 import { augmentCreateTableStatement, getTableName, isCreateTableStatement } from "./table-utils.js";
-import { tables } from "../../sql/tables/tables.js";
+import { sqlTables } from "../../sql/tables/tables.js";
+import { tables } from "./stores.js";
 
 export async function initializeDefaultTables() {
   const results = [];
-  for ( const [ table, query ] of tables ) {
+  for ( const [ table, query ] of sqlTables ) {
+    if ( tables?.has(table) ) continue;
     logger.info('[clickhouse::initializeDefaultTables]\t', `CREATE TABLE [${table}]`);
     results.push({table, query, ...await client.exec({ query })});
   }