From ebfd095cc6043dda9d3e825a15857bbacf8b9a5e Mon Sep 17 00:00:00 2001 From: Denis Carriere Date: Fri, 16 Feb 2024 13:45:48 -0500 Subject: [PATCH 1/2] Add new health queries --- src/fetch/GET.ts | 28 +++++++++++++++------- src/fetch/POST.ts | 1 + src/fetch/blocks.sql | 10 ++++++++ src/fetch/blocks.ts | 51 +++++++++++++--------------------------- src/fetch/cluster.sql | 14 +++++++++++ src/fetch/cluster.ts | 18 +++++++++++++++ src/fetch/openapi.ts | 54 +++++++++++++++++++++++++++++++++++-------- 7 files changed, 123 insertions(+), 53 deletions(-) create mode 100644 src/fetch/blocks.sql create mode 100644 src/fetch/cluster.sql create mode 100644 src/fetch/cluster.ts diff --git a/src/fetch/GET.ts b/src/fetch/GET.ts index b841a33..7b0489c 100644 --- a/src/fetch/GET.ts +++ b/src/fetch/GET.ts @@ -3,21 +3,33 @@ import swaggerFavicon from "../../swagger/favicon.png"; import swaggerHtml from "../../swagger/index.html"; import { metrics } from "../prometheus.js"; import { blocks } from "./blocks.js"; -import { NotFound, toFile, toJSON } from "./cors.js"; +import { NotFound, toFile, toJSON, toText } from "./cors.js"; import { findLatestCursor } from "./cursor.js"; import health from "./health.js"; import { openapi } from "./openapi.js"; +import { cluster } from "./cluster.js"; export default async function (req: Request) { const { pathname } = new URL(req.url); - if (pathname === "/") return toFile(file(swaggerHtml)); - if (pathname === "/favicon.png") return toFile(file(swaggerFavicon)); - if (pathname === "/health") return health(); - if (pathname === "/metrics") return metrics(); - if (pathname === "/openapi") return toJSON(await openapi()); - if (pathname === "/blocks") return blocks(); - if (pathname === "/cursor/latest") return findLatestCursor(req); + try { + if (pathname === "/") return toFile(file(swaggerHtml)); + if (pathname === "/favicon.png") return toFile(file(swaggerFavicon)); + + // queries + if (pathname === "/cursor/latest") return findLatestCursor(req); + + // health + if (pathname === "/blocks") return toJSON(await blocks(req)); + if (pathname === "/cluster") return toJSON(await cluster()); + if (pathname === "/health") return health(); + if (pathname === "/metrics") return metrics(); + + // docs + if (pathname === "/openapi") return toJSON(await openapi()); + } catch (e) { + return toText(String(e), 500); + } return NotFound; } diff --git a/src/fetch/POST.ts b/src/fetch/POST.ts index 5eb74e0..1c36f1b 100644 --- a/src/fetch/POST.ts +++ b/src/fetch/POST.ts @@ -12,6 +12,7 @@ import { query } from "./query.js"; export default async function (req: Request) { const { pathname } = new URL(req.url); + // queries if (pathname === "/query") return query(req); if (pathname === "/hash") return hash(req); diff --git a/src/fetch/blocks.sql b/src/fetch/blocks.sql new file mode 100644 index 0000000..7f20604 --- /dev/null +++ b/src/fetch/blocks.sql @@ -0,0 +1,10 @@ +SELECT + chain, + COUNT() AS count, + COUNTDISTINCT(block_number) AS count_distinct, + MAX(block_number) AS max, + MIN(block_number) AS min, + max - min + 1 AS delta, + delta - count_distinct AS missing +FROM blocks +GROUP BY chain \ No newline at end of file diff --git a/src/fetch/blocks.ts b/src/fetch/blocks.ts index 081e1fb..889d091 100644 --- a/src/fetch/blocks.ts +++ b/src/fetch/blocks.ts @@ -1,16 +1,8 @@ import { z } from "zod"; import client from "../clickhouse/createClient.js"; -import { logger } from "../logger.js"; -import { InternalServerError, toJSON } from "./cors.js"; - -type BlockViewType = Array<{ - count: string; - count_distinct: string; - max: number; - min: number; -}>; export const BlockResponseSchema = z.object({ + chain: z.string(), count: z.number(), distinctCount: z.number(), min: z.number(), @@ -20,32 +12,21 @@ export const BlockResponseSchema = z.object({ }); export type BlockResponseSchema = z.infer; -export async function blocks(): Promise { - const query = ` - SELECT - COUNT() AS count, - COUNTDISTINCT(block_number) AS count_distinct, - MAX(block_number) AS max, - MIN(block_number) AS min - FROM blocks - `; - - try { - const response = await client.query({ query, format: "JSONEachRow" }); - const data = await response.json(); - - const distinctCount = parseInt(data[0].count_distinct); - const max = data[0].max; - const min = data[0].min; - const delta = max - min; - const missing = max - min - distinctCount; - const count = parseInt(data[0].count); +export function getChain(req: Request, required = true) { + const url = new URL(req.url); + const chain = url.searchParams.get("chain"); - const dto: BlockResponseSchema = { max, min, distinctCount, delta, missing, count }; - - return toJSON(dto); - } catch (err) { - logger.error('[blocks]', err); - return InternalServerError; + if (required && !chain) { + throw Error("Missing parameter: chain"); } + return chain; +} + +export async function blocks(req: Request) { + const query = await Bun.file(import.meta.dirname + "/blocks.sql").text() + const chain = getChain(req, false); + const response = await client.query({ query, format: "JSONEachRow" }); + let data = await response.json() as BlockResponseSchema[]; + if ( chain ) data = data.filter((row) => row.chain === chain); + return data; } diff --git a/src/fetch/cluster.sql b/src/fetch/cluster.sql new file mode 100644 index 0000000..73ee7a5 --- /dev/null +++ b/src/fetch/cluster.sql @@ -0,0 +1,14 @@ +SELECT + table, + sum(rows) AS rows, + max(modification_time) AS latest_modification, + formatReadableSize(sum(bytes)) AS data_size, + formatReadableSize(sum(primary_key_bytes_in_memory)) AS primary_keys_size, + any(engine) AS engine, + sum(bytes) AS bytes_size +FROM clusterAllReplicas(default, system.parts) +WHERE active +GROUP BY + database, + table +ORDER BY bytes_size DESC \ No newline at end of file diff --git a/src/fetch/cluster.ts b/src/fetch/cluster.ts new file mode 100644 index 0000000..f36c49c --- /dev/null +++ b/src/fetch/cluster.ts @@ -0,0 +1,18 @@ +import { z } from "zod"; +import client from "../clickhouse/createClient.js"; + +export const ClusterSchema = z.object({ + count: z.number(), + distinctCount: z.number(), + min: z.number(), + max: z.number(), + delta: z.number(), + missing: z.number(), +}); +export type ClusterSchema = z.infer; + +export async function cluster() { + const query = await Bun.file(import.meta.dirname + "/cluster.sql").text() + const response = await client.query({ query, format: "JSONEachRow" }); + return response.json(); +} diff --git a/src/fetch/openapi.ts b/src/fetch/openapi.ts index 21b61cf..27e7cc1 100644 --- a/src/fetch/openapi.ts +++ b/src/fetch/openapi.ts @@ -1,20 +1,21 @@ import pkg from "../../package.json" assert { type: "json" }; import { LicenseObject } from "openapi3-ts/oas30"; -import { OpenApiBuilder, ResponsesObject, SchemaObject } from "openapi3-ts/oas31"; +import { OpenApiBuilder, ParameterObject, ResponsesObject, SchemaObject } from "openapi3-ts/oas31"; import { z } from "zod"; import * as ztjs from "zod-to-json-schema"; import { store } from "../clickhouse/stores.js"; import { BodySchema } from "../schemas.js"; import { BlockResponseSchema } from "./blocks.js"; +import { ClusterSchema } from "./cluster.js"; const zodToJsonSchema = (...params: Parameters<(typeof ztjs)["zodToJsonSchema"]>) => ztjs.zodToJsonSchema(...params) as SchemaObject; const TAGS = { + QUERIES: "Queries", USAGE: "Usage", MAINTENANCE: "Maintenance", - QUERIES: "Queries", HEALTH: "Health", DOCS: "Documentation", } as const; @@ -36,6 +37,24 @@ const PUT_RESPONSES: ResponsesObject = { const ExecuteSchemaResponse = z.object({ success: z.literal("OK"), schema: z.string() }); +async function paramChain(required = true): Promise { + return { + name: "chain", + in: "query", + required, + schema: { enum: await store.chains }, + }; +} + +async function paramModuleHash(required = true): Promise { + return { + name: "module_hash", + in: "query", + required: true, + schema: { enum: await store.moduleHashes }, + }; +} + export async function openapi() { return new OpenApiBuilder() .addInfo({ @@ -205,13 +224,8 @@ export async function openapi() { tags: [TAGS.QUERIES], summary: "Finds the latest cursor for a given chain and table", parameters: [ - { name: "chain", in: "query", required: true, schema: { enum: await store.chains } }, - { - name: "module_hash", - in: "query", - required: true, - schema: { enum: await store.moduleHashes }, - }, + await paramChain(true), + await paramModuleHash(true) ], responses: { 200: { @@ -233,8 +247,11 @@ export async function openapi() { }) .addPath("/blocks", { get: { - tags: [TAGS.QUERIES], + tags: [TAGS.HEALTH], summary: "Gives a summary of known blocks, including min, max and unique block numbers", + parameters: [ + await paramChain(false) + ], responses: { 200: { description: "Block number summary", @@ -248,6 +265,23 @@ export async function openapi() { }, }, }) + .addPath("/cluster", { + get: { + tags: [TAGS.HEALTH], + summary: "Global overview of your cluster", + responses: { + 200: { + description: "Global overview of your cluster", + content: { + "application/json": { + schema: zodToJsonSchema(ClusterSchema), + }, + }, + }, + 500: { description: "Internal server errror" }, + }, + }, + }) .addPath("/query", { post: { tags: [TAGS.QUERIES], From 776116e971b01496682926a0feb36c9fad588bd9 Mon Sep 17 00:00:00 2001 From: Denis Carriere Date: Fri, 16 Feb 2024 19:51:37 -0500 Subject: [PATCH 2/2] Update table import in table-initialization.ts --- src/clickhouse/table-initialization.ts | 2 +- src/clickhouse/tables/blocks.sql | 3 ++- src/clickhouse/tables/blocks_mv.sql | 22 ++++++++++++++++++++++ src/clickhouse/tables/index.ts | 19 ------------------- src/clickhouse/tables/tables.ts | 14 ++++++++++++++ 5 files changed, 39 insertions(+), 21 deletions(-) create mode 100644 src/clickhouse/tables/blocks_mv.sql delete mode 100644 src/clickhouse/tables/index.ts create mode 100644 src/clickhouse/tables/tables.ts diff --git a/src/clickhouse/table-initialization.ts b/src/clickhouse/table-initialization.ts index 1c80d98..4dec1cb 100644 --- a/src/clickhouse/table-initialization.ts +++ b/src/clickhouse/table-initialization.ts @@ -2,7 +2,7 @@ import { logger } from "../logger.js"; import { Err, Ok, Result } from "../result.js"; import client from "./createClient.js"; import { augmentCreateTableStatement, getTableName, isCreateTableStatement } from "./table-utils.js"; -import tables from "./tables/index.js"; +import tables from "./tables/tables.js"; export async function initializeDefaultTables(): Promise { const promiseResults = await Promise.allSettled( diff --git a/src/clickhouse/tables/blocks.sql b/src/clickhouse/tables/blocks.sql index c0403aa..2bdb7d5 100644 --- a/src/clickhouse/tables/blocks.sql +++ b/src/clickhouse/tables/blocks.sql @@ -1,3 +1,4 @@ +-- blocks -- CREATE TABLE IF NOT EXISTS blocks ( block_id FixedString(64), block_number UInt32(), @@ -6,4 +7,4 @@ CREATE TABLE IF NOT EXISTS blocks ( ) ENGINE = ReplacingMergeTree PRIMARY KEY (block_id) -ORDER BY (block_id, chain, block_number, timestamp); \ No newline at end of file +ORDER BY (block_id, chain, block_number, timestamp); diff --git a/src/clickhouse/tables/blocks_mv.sql b/src/clickhouse/tables/blocks_mv.sql new file mode 100644 index 0000000..9d1cac7 --- /dev/null +++ b/src/clickhouse/tables/blocks_mv.sql @@ -0,0 +1,22 @@ +-- view -- +CREATE MATERIALIZED VIEW IF NOT EXISTS blocks_mv +ENGINE = MergeTree +ORDER BY (chain, timestamp, block_number) +AS SELECT + block_id, + block_number, + chain, + timestamp +FROM blocks; + +-- DROP TABLE IF EXISTS blocks_mv; + +-- OPTIMIZE TABLE blocks_mv FINAL; + +-- -- insert -- +-- INSERT INTO blocks_mv SELECT +-- block_id, +-- block_number, +-- chain, +-- timestamp +-- FROM blocks; \ No newline at end of file diff --git a/src/clickhouse/tables/index.ts b/src/clickhouse/tables/index.ts deleted file mode 100644 index b9b50b1..0000000 --- a/src/clickhouse/tables/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -import blocks_sql from "./blocks.sql"; -import deleted_entity_changes_sql from "./deleted_entity_changes.sql"; -import final_blocks_sql from "./final_blocks.sql"; -import module_hashes_sql from "./module_hashes.sql"; -import unparsed_json_sql from "./unparsed_json.sql"; - -export const blocks = await Bun.file(blocks_sql).text(); -export const final_blocks = await Bun.file(final_blocks_sql).text(); -export const module_hashes = await Bun.file(module_hashes_sql).text(); -export const unparsed_json = await Bun.file(unparsed_json_sql).text(); -export const deleted_entity_changes = await Bun.file(deleted_entity_changes_sql).text(); - -export default [ - ["blocks", blocks], - ["final_blocks", final_blocks], - ["module_hashes", module_hashes], - ["unparsed_json", unparsed_json], - ["deleted_entity_changes", deleted_entity_changes], -]; diff --git a/src/clickhouse/tables/tables.ts b/src/clickhouse/tables/tables.ts new file mode 100644 index 0000000..29024e5 --- /dev/null +++ b/src/clickhouse/tables/tables.ts @@ -0,0 +1,14 @@ +import { join } from "path"; + +function file(path: string) { + return Bun.file(join(import.meta.dirname, path)).text(); +} + +const glob = new Bun.Glob("**/*.sql"); +const tables: [string, string][] = []; + +for (const path of glob.scanSync(import.meta.dirname)) { + tables.push([path.replace(".sql", ""), await file(path)]) +} + +export default tables; \ No newline at end of file