diff --git a/.env b/.env deleted file mode 100644 index 5ea1c32..0000000 --- a/.env +++ /dev/null @@ -1,4 +0,0 @@ -DB_HOST=http://127.0.0.1:8123 -DB_NAME=clickhouse_sink -DB_USERNAME=default -DB_PASSWORD= diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..8948dc9 --- /dev/null +++ b/.env.example @@ -0,0 +1,13 @@ +# API Server +PORT=8080 +HOSTNAME=localhost + +# Clickhouse Database +HOST=http://127.0.0.1:8123 +DATABASE=default +USERNAME=default +PASSWORD= +MAX_LIMIT=500 + +# Logging +VERBOSE=true \ No newline at end of file diff --git a/.github/workflows/bun-build.yml b/.github/workflows/bun-build.yml index 65159d4..021ffe8 100644 --- a/.github/workflows/bun-build.yml +++ b/.github/workflows/bun-build.yml @@ -1,4 +1,4 @@ -name: Build +name: Generate standalone Bun executable on: release: types: [published] @@ -7,21 +7,17 @@ permissions: contents: write jobs: - bun-build: + build-and-push-release: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: oven-sh/setup-bun@v1 with: bun-version: latest - - - name: "Install Dependencies" - run: bun install - - - name: "Build app" - run: bun run build - + - run: bun install + - run: bun run test + - run: bun run build - uses: softprops/action-gh-release@v1 with: files: | - substreams-erc20-api + substreams-erc20-api \ No newline at end of file diff --git a/.github/workflows/bun-lint.yml b/.github/workflows/bun-lint.yml deleted file mode 100644 index 791b3e8..0000000 --- a/.github/workflows/bun-lint.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Lint - -on: - push: - branches: [develop*, main] - pull_request: - branches: [develop*, main] - -jobs: - bun-lint: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Install bun - uses: oven-sh/setup-bun@v1 - - - name: "Install Dependencies" - run: | - bun install - - - name: "Lint" - run: bun lint diff --git a/.github/workflows/bun-test.yml b/.github/workflows/bun-test.yml index c41cf32..7d5782b 100644 --- a/.github/workflows/bun-test.yml +++ b/.github/workflows/bun-test.yml @@ -1,51 +1,14 @@ -name: Test +name: Bun Test -on: - push: - branches: [develop*, main] - pull_request: - branches: [develop*, main] +on: push jobs: - bun-test: + build-and-test: runs-on: ubuntu-latest - environment: dev-test steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Install bun - uses: oven-sh/setup-bun@v1 - - - name: "Install Dependencies" - run: | - bun install - - - name: "Setup local Clickhouse DB" - uses: metrico/clickhouse-server-action@v1.0.0 - - - name: "Insert mock data into Clickhouse DB for testing" - run: | - curl https://clickhouse.com/ | sh - echo "CREATE DATABASE ${{ secrets.DB_NAME }}" > create.sql - ./clickhouse client --queries-file ./create.sql - echo "USE ${{ secrets.DB_NAME }}; CREATE TABLE TotalSupply (address FixedString(40), supply Nullable(String),id String , block_id FixedString(64), module_hash FixedString(40),chain LowCardinality(String) ) ENGINE = MergeTree() ORDER BY (address); CREATE TABLE Contracts (address FixedString(40), name Nullable(String), symbol Nullable(String), decimals Nullable(String), id String , block_id FixedString(64), module_hash FixedString(40),chain LowCardinality(String)) ENGINE = MergeTree() ORDER BY (address); CREATE TABLE balance_changes (contract FixedString(40), owner Nullable(String), old_balance Nullable(String), new_balance Nullable(String), transaction_id Nullable(String), id String , block_id FixedString(64), module_hash FixedString(40),chain LowCardinality(String)) ENGINE = MergeTree() ORDER BY (contract); CREATE TABLE IF NOT EXISTS manifest ( module_hash FixedString(40), module_name String(), chain LowCardinality(String), type String(), ) ENGINE = ReplacingMergeTree PRIMARY KEY (module_hash) ORDER BY (module_hash, module_name); CREATE TABLE IF NOT EXISTS block ( block_id FixedString(64), block_number UInt32(), chain LowCardinality(String), timestamp DateTime64(3, 'UTC'), final_block Bool, ) ENGINE = ReplacingMergeTree PRIMARY KEY (block_id) ORDER BY (block_id, block_number, timestamp);" > setup.sql - ./clickhouse client --queries-file ./setup.sql - echo "INSERT INTO ${{ secrets.DB_NAME }}.block (block_id, block_number, chain, timestamp,final_block) VALUES ('0006b5036d09b082543fe4306d98f6ad9b438fa4c139124106018fdfc90ad38f', 15010581, 'eth', '2022-06-23 01:43:01.000',true);" > block.sql - ./clickhouse client --queries-file ./block.sql - echo "INSERT INTO ${{ secrets.DB_NAME }}.TotalSupply (address, supply, id, block_id,module_hash,chain) VALUES ('cb9df5dc2ed5d7d3972f601acfe35cdbe57341e0', '100000', '00000000000045166c45af0fc6e4cf31d9e14b9a', '0006b5036d09b082543fe4306d98f6ad9b438fa4c139124106018fdfc90ad38f','9a8fdd0be04bbe23f68ed4c38a02f7cbc96b86fb','eth');" > insert1.sql - ./clickhouse client --queries-file ./insert1.sql - echo "INSERT INTO ${{ secrets.DB_NAME }}.Contracts (address, name, symbol, decimals,id, block_id,module_hash,chain) VALUES ('cb9df5dc2ed5d7d3972f601acfe35cdbe57341e0', 'wing', '$wing', '8','00000000000045166c45af0fc6e4cf31d9e14b9a', '0006b5036d09b082543fe4306d98f6ad9b438fa4c139124106018fdfc90ad38f','9a8fdd0be04bbe23f68ed4c38a02f7cbc96b86fb','eth');" > insert2.sql - ./clickhouse client --queries-file ./insert2.sql - echo "INSERT INTO ${{ secrets.DB_NAME }}.balance_changes (contract, owner, old_balance, new_balance, transaction_id,id, block_id,module_hash,chain) VALUES ('c083e9947Cf02b8FfC7D3090AE9AEA72DF98FD47', '39fA8c5f2793459D6622857E7D9FbB4BD91766d3', '123123', '3423443', 'ab3612eed62a184eed2ae86bcad766183019cf40f82e5316f4d7c4e61f4baa44','00000000000045166c45af0fc6e4cf31d9e14b9a', '0006b5036d09b082543fe4306d98f6ad9b438fa4c139124106018fdfc90ad38f','9a8fdd0be04bbe23f68ed4c38a02f7cbc96b86fb','eth');" > insert3.sql - ./clickhouse client --queries-file ./insert3.sql - - name: "Run test" - run: | - bun test - env: - HOSTNAME: ${{ vars.HOST}} - PORT: ${{ vars.PORT }} - DB_HOST: ${{ vars.DB_HOST }} - DB_NAME: ${{ secrets.DB_NAME }} - DB_USERNAME: ${{ secrets.DB_USERNAME }} - DB_PASSWORD: "" + - uses: actions/checkout@v3 + - uses: oven-sh/setup-bun@v1 + with: + bun-version: latest + - run: bun install + - run: bun run test \ No newline at end of file diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index df46f88..660cf83 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -1,14 +1,14 @@ name: GitHub Container Registry on: release: - types: [published] + types: [ published ] env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} jobs: - ghcr: + build-and-push-image: runs-on: ubuntu-latest permissions: contents: read @@ -39,4 +39,4 @@ jobs: context: . push: true tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index bfc63f8..9f0abaf 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ yarn-debug.log* yarn-error.log* lerna-debug.log* .pnpm-debug.log* +package-lock.json +.env # Diagnostic reports (https://nodejs.org/api/report.html) report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json diff --git a/README.md b/README.md index 85b758e..dd6b9ca 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,26 @@ -``` -bun install -bun run dev -``` +# [`Substreams`](https://substreams.streamingfast.io/) ERC20 API -``` -open http://localhost:8080 -``` +[![.github/workflows/bun-test.yml](https://github.com/pinax-network/substreams-erc20-api/actions/workflows/bun-test.yml/badge.svg)](https://github.com/pinax-network/substreams-erc20-api/actions/workflows/bun-test.yml) - ███████╗██████╗░░█████╗░██████╗░░█████╗░  ░█████╗░██████╗░██╗ - ██╔════╝██╔══██╗██╔══██╗╚════██╗██╔══██╗  ██╔══██╗██╔══██╗██║ - █████╗░░██████╔╝██║░░╚═╝░░███╔═╝██║░░██║  ███████║██████╔╝██║ - ██╔══╝░░██╔══██╗██║░░██╗██╔══╝░░██║░░██║  ██╔══██║██╔═══╝░██║ - ███████╗██║░░██║╚█████╔╝███████╗╚█████╔╝  ██║░░██║██║░░░░░██║ - ╚══════╝╚═╝░░╚═╝░╚════╝░╚══════╝░╚════╝░  ╚═╝░░╚═╝╚═╝░░░░░╚═╝ - -# [`Substreams`](https://substreams.streamingfast.io/) ERC20 API +> ERC-20 Balance, Supply, Contract API ## REST API -| Pathname | Description | -| ----------------------------------------------------------------------------------------------- | ------------------------------------------------------- | -| GET `/` | Banner | -| GET `/supply?address=&block=` | Returns the total supply of a contract | -| GET `/contract?address=` | Returns Contract information (name,symbol,decimals) | -| GET `/balance?wallet?&address=&block=` | Returns the wallet balance | -| GET `/openapi` | [OpenAPI v3 JSON](https://spec.openapis.org/oas/v3.0.0) | -| GET `/swagger` | [Swagger UI](https://swagger.io/resources/open-api/) | +| Pathname | Description | +| ----------------------|-------------------------------------------------------- | +| GET `/` | [Swagger UI](https://swagger.io/resources/open-api/) +| GET `/chains` | Available `chains` +| GET `/supply` | ERC20 total supply +| GET `/contract` | ERC20 contract information (name,symbol,decimals) +| GET `/balance` | ERC20 balance changes +| GET `/health` | Health check +| GET `/metrics` | Prometheus metrics +| GET `/openapi` | [OpenAPI v3 JSON](https://spec.openapis.org/oas/v3.0.0) ## Requirements -- [Clickhouse](clickhouse.com/) - -Additionnaly to pull data directly from a substream: - -- [Substreams Sink Clickhouse](https://github.com/pinax-network/substreams-sink-clickhouse/) +- [ClickHouse](clickhouse.com/) +- [Substreams Sink ClickHouse](https://github.com/pinax-network/substreams-sink-clickhouse/) ## Quickstart @@ -54,13 +41,19 @@ $ chmod +x ./substreams-erc20-api ## `.env` Environment variables ```env -# Optional +# API Server PORT=8080 HOSTNAME=localhost -DB_HOST=http://localhost:8123 -DB_NAME=demo -DB_USERNAME=default -DB_PASSWORD= + +# Clickhouse Database +HOST=http://127.0.0.1:8123 +DATABASE=default +USERNAME=default +PASSWORD= +MAX_LIMIT=500 + +# Logging +VERBOSE=true ``` ## Help @@ -69,19 +62,19 @@ DB_PASSWORD= $ ./substreams-erc20-api --help Usage: substreams-erc20-api [options] -Timestamps <> Block numbers conversion for your favorite chains +ERC20 API powered by Substreams Options: - --port Server listen on HTTP port (default: "8080", env: PORT) - --hostname Server listen on HTTP hostname (default: "localhost", env: HOST) - --db-host Clickhouse DB HTTP hostname (default: "http://localhost:8123", env: dbHost) - --name Clickhouse DB table name (default: "demo", env: DB_NAME) - --username Clickhouse DB username (default: "default", env: DB_USERNAME) - --password Clickhouse DB password (default: "", env: DB_PASSWORD) - --max-elements-queried Maximum number of query elements when using arrays as parameters (default: 10, env: MAX_ELEMENTS_QUERIED) - --verbose Enable verbose logging (default: false, env: VERBOSE) - -V, --version output the version number - -h, --help display help for command + -V, --version output the version number + -p, --port HTTP port on which to attach the API (default: "8080", env: PORT) + -v, --verbose Enable verbose logging (choices: "true", "false", default: false, env: VERBOSE) + --hostname Server listen on HTTP hostname (default: "localhost", env: HOSTNAME) + --host Database HTTP hostname (default: "http://localhost:8123", env: HOST) + --username Database user (default: "default", env: USERNAME) + --password Password associated with the specified username (default: "", env: PASSWORD) + --database The database to use inside ClickHouse (default: "default", env: DATABASE) + --max-limit Maximum LIMIT queries (default: 10000, env: MAX_LIMIT) + -h, --help display help for command ``` ## Docker environment diff --git a/bun.lockb b/bun.lockb index 8ffcda2..9045f00 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/index.ts b/index.ts new file mode 100644 index 0000000..330e5d3 --- /dev/null +++ b/index.ts @@ -0,0 +1,19 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/index.ts +import { config } from "./src/config"; +import { logger } from "./src/logger"; +import GET from "./src/fetch/GET"; +import * as prometheus from "./src/prometheus.js"; + +if (config.verbose) logger.enable(); + +const app = Bun.serve({ + hostname: config.hostname, + port: config.port, + fetch(req: Request) { + if (req.method === "GET") return GET(req); + prometheus.request_error.inc({pathname: new URL(req.url).pathname, status: 400}); + return new Response("Invalid request", { status: 400 }); + } +}); + +logger.info(`Server listening on http://${app.hostname}:${app.port}`); \ No newline at end of file diff --git a/package.json b/package.json index 1e2bcea..2c83708 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,9 @@ "description": "ERC20 API powered by Substreams", "version": "v0.0.1", "name": "substreams-erc20-api", - "repository": "https://github.com/pinax-network/substreams-erc20-api", + "type": "module", + "license": "MIT", + "homepage": "https://github.com/pinax-network/substreams-erc20-api", "authors": [ { "name": "Mathieu Lefebvre", @@ -16,25 +18,24 @@ } ], "scripts": { - "dev": "bun run --watch src/index.ts", - "lint": "bun run tsc --noEmit --skipLibCheck --pretty", - "test": "bun test test/*.spec.ts --coverage", - "build": "bun build --compile ./src/index.ts --outfile substreams-erc20-api" + "start": "bun index.ts", + "dev": "bun --watch index.ts", + "pretest": "bunx tsc --noEmit", + "test": "bun test", + "build": "bun build --compile ./index.ts --outfile substreams-erc20-api" }, - "license": "MIT", "dependencies": { - "@clickhouse/client-web": "^0.2.2", - "@hono/zod-openapi": "^0.7.2", - "@sinclair/typebox": "^0.31.17", - "commander": "^11.1.0", - "dotenv": "^16.3.1", - "ethers": "^6.8.0", - "hono": "^3.7.2", - "tslog": "^4.9.2", - "zod": "^3.22.4" + "@clickhouse/client-web": "latest", + "commander": "latest", + "dotenv": "latest", + "ethers": "latest", + "openapi3-ts": "latest", + "prom-client": "latest", + "tslog": "latest", + "zod": "latest" }, "devDependencies": { - "bun-types": "^0.6.2", - "typescript": "^5.2.2" + "bun-types": "latest", + "typescript": "latest" } } diff --git a/src/clickhouse/createClient.ts b/src/clickhouse/createClient.ts new file mode 100644 index 0000000..cd732b5 --- /dev/null +++ b/src/clickhouse/createClient.ts @@ -0,0 +1,20 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/clickhouse/createClient.ts +import { createClient } from "@clickhouse/client-web"; +import { ping } from "./ping"; +import { APP_NAME, config } from "../config"; + +const client = createClient({ + ...config, + clickhouse_settings: { + allow_experimental_object_type: 1, + }, + application: APP_NAME, +}) + +// These overrides should not be required but the @clickhouse/client-web instance +// does not work well with Bun's implementation of Node streams. +// https://github.com/oven-sh/bun/issues/5470 +client.command = client.exec; +client.ping = ping; + +export default client; \ No newline at end of file diff --git a/src/clickhouse/makeQuery.ts b/src/clickhouse/makeQuery.ts new file mode 100644 index 0000000..5cf7018 --- /dev/null +++ b/src/clickhouse/makeQuery.ts @@ -0,0 +1,30 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/clickhouse/makeQuery.ts +import { logger } from "../logger"; +import * as prometheus from "../prometheus"; +import client from "./createClient"; + +export interface Meta { + name: string, + type: string +} +export interface Query { + meta: Meta[], + data: T[], + rows: number, + statistics: { + elapsed: number, + rows_read: number, + bytes_read: number, + } +} + +export async function makeQuery(query: string) { + const response = await client.query({ query }) + const data: Query = await response.json(); + prometheus.query.inc(); + prometheus.bytes_read.inc(data.statistics.bytes_read); + prometheus.rows_read.inc(data.statistics.rows_read); + prometheus.elapsed.inc(data.statistics.elapsed); + logger.info({ query, statistics: data.statistics, rows: data.rows }); + return data; +} \ No newline at end of file diff --git a/src/clickhouse/ping.ts b/src/clickhouse/ping.ts new file mode 100644 index 0000000..1d69880 --- /dev/null +++ b/src/clickhouse/ping.ts @@ -0,0 +1,14 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/clickhouse/ping.ts +import { PingResult } from "@clickhouse/client-web"; +import client from "./createClient"; + +// Does not work with Bun's implementation of Node streams. +export async function ping(): Promise { + try { + await client.exec({ query: "SELECT 1" }); + return { success: true }; + } catch (err) { + const message = typeof err === "string" ? err : JSON.stringify(err); + return { success: false, error: new Error(message) }; + } +}; \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index dd65b04..61f5471 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,52 +1,44 @@ -import { z } from '@hono/zod-openapi'; import "dotenv/config"; -import { Command, Option } from "commander"; +import { z } from 'zod'; +import { Option, program } from "commander"; import pkg from "../package.json"; export const DEFAULT_PORT = "8080"; export const DEFAULT_HOSTNAME = "localhost"; -export const DEFAULT_DB_HOST = "http://localhost:8123"; -export const DEFAULT_DB_NAME = "clickhouse_sink"; -export const DEFAULT_DB_USERNAME = "default"; -export const DEFAULT_DB_PASSWORD = ""; -export const DEFAULT_MAX_ELEMENTS_QUERIES = 10; -export const DEFAULT_VERBOSE = true; - -const CommanderSchema = z.object({ - NODE_ENV: z.string().optional(), - port: z.string().default(DEFAULT_PORT), - hostname: z.string().default(DEFAULT_HOSTNAME), - dbHost: z.string().default(DEFAULT_DB_HOST), - name: z.string().default(DEFAULT_DB_NAME), - username: z.string().default(DEFAULT_DB_USERNAME), - password: z.string().default(DEFAULT_DB_PASSWORD), - maxElementsQueried: z.coerce.number().default(DEFAULT_MAX_ELEMENTS_QUERIES).describe( - 'Maximum number of query elements when using arrays as parameters' - ), - verbose: z.boolean().default(DEFAULT_VERBOSE), -}); - -export function decode(data: unknown) { - return CommanderSchema.passthrough().parse(data); // throws on failure -} +export const DEFAULT_HOST = "http://localhost:8123"; +export const DEFAULT_DATABASE = "default"; +export const DEFAULT_USERNAME = "default"; +export const DEFAULT_PASSWORD = ""; +export const DEFAULT_MAX_LIMIT = 10000; +export const DEFAULT_VERBOSE = false; +export const DEFAULT_SORT_BY = "DESC"; +export const APP_NAME = pkg.name; // parse command line options -const opts = new Command() - .name(pkg.name) - .description(pkg.description) - .showHelpAfterError() - .addOption(new Option("--port ", "Server listen on HTTP port").default(DEFAULT_PORT).env("PORT")) - .addOption(new Option("--hostname ", "Server listen on HTTP hostname").default(DEFAULT_HOSTNAME).env("HOST")) - .addOption(new Option("--db-host ", "Clickhouse DB HTTP hostname").default(DEFAULT_DB_HOST).env("DB_HOST")) - .addOption(new Option("--name ", "Clickhouse DB table name").default(DEFAULT_DB_NAME).env("DB_NAME")) - .addOption(new Option("--username ", "Clickhouse DB username").default(DEFAULT_DB_USERNAME).env("DB_USERNAME")) - .addOption(new Option("--password ", "Clickhouse DB password").default(DEFAULT_DB_PASSWORD).env("DB_PASSWORD")) - .addOption(new Option("--max-elements-queried ", "Maximum number of query elements when using arrays as parameters") - .default(DEFAULT_MAX_ELEMENTS_QUERIES).env("MAX_ELEMENTS_QUERIED")) - .addOption(new Option("--verbose ", "Enable verbose logging").default(DEFAULT_VERBOSE).env("VERBOSE")) // TODO: Use verbose logging - .version(pkg.version) - .parse(process.argv).opts(); +const opts = program + .name(pkg.name) + .version(pkg.version) + .description(pkg.description) + .showHelpAfterError() + .addOption(new Option("-p, --port ", "HTTP port on which to attach the API").env("PORT").default(DEFAULT_PORT)) + .addOption(new Option("-v, --verbose ", "Enable verbose logging").choices(["true", "false"]).env("VERBOSE").default(DEFAULT_VERBOSE)) + .addOption(new Option("--hostname ", "Server listen on HTTP hostname").env("HOSTNAME").default(DEFAULT_HOSTNAME)) + .addOption(new Option("--host ", "Database HTTP hostname").env("HOST").default(DEFAULT_HOST)) + .addOption(new Option("--username ", "Database user").env("USERNAME").default(DEFAULT_USERNAME)) + .addOption(new Option("--password ", "Password associated with the specified username").env("PASSWORD").default(DEFAULT_PASSWORD)) + .addOption(new Option("--database ", "The database to use inside ClickHouse").env("DATABASE").default(DEFAULT_DATABASE)) + .addOption(new Option("--max-limit ", "Maximum LIMIT queries").env("MAX_LIMIT").default(DEFAULT_MAX_LIMIT)) + .parse() + .opts(); -let config: z.infer = decode({ ...opts, ...process.env }); -export default config!; \ No newline at end of file +export const config = z.object({ + port: z.string(), + hostname: z.string(), + host: z.string(), + database: z.string(), + username: z.string(), + password: z.string(), + maxLimit: z.coerce.number(), + verbose: z.coerce.boolean(), +}).parse(opts); \ No newline at end of file diff --git a/src/fetch/GET.ts b/src/fetch/GET.ts new file mode 100644 index 0000000..03aa0fa --- /dev/null +++ b/src/fetch/GET.ts @@ -0,0 +1,26 @@ +import { registry } from "../prometheus.js"; +import openapi from "./openapi.js"; +import health from "./health.js"; +import chains from "./chains.js"; +import balance from "./balance.js"; +import contract from "./contract.js"; +import supply from "./supply.js"; +import * as prometheus from "../prometheus.js"; +import { logger } from "../logger.js"; + +export default async function (req: Request) { + const { pathname} = new URL(req.url); + prometheus.request.inc({pathname}); + if ( pathname === "/" ) return new Response(Bun.file("./swagger/index.html")); + if ( pathname === "/favicon.png" ) return new Response(Bun.file("./swagger/favicon.png")); + if ( pathname === "/health" ) return health(req); + if ( pathname === "/metrics" ) return new Response(await registry.metrics(), {headers: {"Content-Type": registry.contentType}}); + if ( pathname === "/openapi" ) return new Response(openapi, {headers: {"Content-Type": "application/json"}}); + if ( pathname === "/chains" ) return chains(req); + if ( pathname === "/supply" ) return supply(req); + if ( pathname === "/balance" ) return balance(req); + if ( pathname === "/contract" ) return contract(req); + logger.warn(`Not found: ${pathname}`); + prometheus.request_error.inc({pathname, status: 404}); + return new Response("Not found", { status: 404 }); +} \ No newline at end of file diff --git a/src/fetch/balance.ts b/src/fetch/balance.ts new file mode 100644 index 0000000..d1f19eb --- /dev/null +++ b/src/fetch/balance.ts @@ -0,0 +1,20 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/fetch/block.ts +import { makeQuery } from "../clickhouse/makeQuery.js"; +import { logger } from "../logger.js"; +import { getBalanceChanges } from "../queries.js"; +import * as prometheus from "../prometheus.js"; +import { toJSON } from "./utils.js"; + +export default async function (req: Request) { + try { + const { searchParams } = new URL(req.url); + logger.info({searchParams: Object.fromEntries(Array.from(searchParams))}); + const query = await getBalanceChanges(searchParams); + const response = await makeQuery(query) + return toJSON(response.data); + } catch (e: any) { + logger.error(e); + prometheus.request_error.inc({pathname: "/balance", status: 400}); + return new Response(e.message, { status: 400 }); + } +} \ No newline at end of file diff --git a/src/fetch/chains.ts b/src/fetch/chains.ts new file mode 100644 index 0000000..7b33c0e --- /dev/null +++ b/src/fetch/chains.ts @@ -0,0 +1,22 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/fetch/chains.ts +import { makeQuery } from "../clickhouse/makeQuery.js"; +import { logger } from "../logger.js"; +import * as prometheus from "../prometheus.js"; +import { getChain } from "../queries.js"; +import { toJSON } from "./utils.js"; + +export async function supportedChainsQuery() { + const response = await makeQuery<{chain: string}>(getChain()); + return response.data.map((r) => r.chain); +} + +export default async function (req: Request) { + try { + const chains = await supportedChainsQuery(); + return toJSON(chains); + } catch (e: any) { + logger.error(e); + prometheus.request_error.inc({pathname: "/chains", status: 400}); + return new Response(e.message, { status: 400 }); + } +} \ No newline at end of file diff --git a/src/fetch/contract.ts b/src/fetch/contract.ts new file mode 100644 index 0000000..e734e4e --- /dev/null +++ b/src/fetch/contract.ts @@ -0,0 +1,20 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/fetch/block.ts +import { makeQuery } from "../clickhouse/makeQuery.js"; +import { logger } from "../logger.js"; +import { getContracts } from "../queries.js"; +import * as prometheus from "../prometheus.js"; +import { toJSON } from "./utils.js"; + +export default async function (req: Request) { + try { + const { searchParams } = new URL(req.url); + logger.info({searchParams: Object.fromEntries(Array.from(searchParams))}); + const query = await getContracts(searchParams); + const response = await makeQuery(query) + return toJSON(response.data); + } catch (e: any) { + logger.error(e); + prometheus.request_error.inc({pathname: "/contract", status: 400}); + return new Response(e.message, { status: 400 }); + } +} \ No newline at end of file diff --git a/src/fetch/health.ts b/src/fetch/health.ts new file mode 100644 index 0000000..e74db6e --- /dev/null +++ b/src/fetch/health.ts @@ -0,0 +1,17 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/fetch/health.ts +import client from "../clickhouse/createClient.js"; +import { logger } from "../logger.js"; +import * as prometheus from "../prometheus.js"; + +export default async function (req: Request) { + try { + const response = await client.ping(); + if (response.success === false) throw new Error(response.error.message); + if (response.success === true ) return new Response("OK"); + return new Response("Unknown response from ClickHouse"); + } catch (e: any) { + logger.error(e); + prometheus.request_error.inc({ pathname: "/health", status: 500}); + return new Response(e.message, { status: 500 }); + } +} \ No newline at end of file diff --git a/src/fetch/openapi.ts b/src/fetch/openapi.ts new file mode 100644 index 0000000..bf96b22 --- /dev/null +++ b/src/fetch/openapi.ts @@ -0,0 +1,160 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/fetch/openapi.ts +import pkg from "../../package.json" assert { type: "json" }; + +import { OpenApiBuilder, SchemaObject, ExampleObject, ParameterObject } from "openapi3-ts/oas31"; +import { config } from "../config"; +import { getBalanceChanges, getContracts, getTotalSupply } from "../queries"; +import { registry } from "../prometheus"; +import { makeQuery } from "../clickhouse/makeQuery"; +import { supportedChainsQuery } from "./chains.js"; + +const TAGS = { + MONITORING: "Monitoring", + HEALTH: "Health", + USAGE: "Usage", + DOCS: "Documentation", +} as const; + +const chains = await supportedChainsQuery(); +// const supply_example = (await makeQuery(await getTotalSupply( new URLSearchParams({limit: "1"})))).data; +// const contract_example = (await makeQuery(await getContracts( new URLSearchParams({limit: "1"})))).data; +// const balance_example = (await makeQuery(await getBalanceChanges( new URLSearchParams({limit: "1"})))).data; + +// TO-DO: make dynamic examples +const supply_example = {}; +const contract_example = {}; +const balance_example = {}; + +// TO-DO: apply timestamp filters to docs +// https://github.com/pinax-network/substreams-erc20-api/issues/4 +const timestampSchema: SchemaObject = { anyOf: [ + {type: "number"}, + {type: "string", format: "date"}, + {type: "string", format: "date-time"} + ] +}; +const timestampExamples: ExampleObject = { + unix: { summary: `Unix Timestamp (seconds)` }, + date: { summary: `Full-date notation`, value: '2023-10-18' }, + datetime: { summary: `Date-time notation`, value: '2023-10-18T00:00:00Z'}, +} + +const parameterChain: ParameterObject = { + name: "chain", + in: "query", + description: "Filter by chain", + required: false, + schema: {enum: chains}, +} +const parameterString = (name: string = "address", required = false) => ({ + name, + in: "query", + description: `Filter by ${name}`, + required, + schema: {type: "string"}, +} as ParameterObject); + +const parameterLimit: ParameterObject = { + name: "limit", + in: "query", + description: "Used to specify the number of records to return.", + required: false, + schema: { type: "number", maximum: config.maxLimit, minimum: 1 }, +} + +export default new OpenApiBuilder() + .addInfo({ + title: pkg.name, + version: pkg.version, + description: pkg.description, + license: {name: pkg.license}, + }) + .addExternalDocs({ url: pkg.homepage, description: "Extra documentation" }) + .addSecurityScheme("auth-key", { type: "http", scheme: "bearer" }) + .addPath("/chains", { + get: { + tags: [TAGS.USAGE], + summary: 'Supported chains', + responses: { + 200: { + description: "Array of chains", + content: { + "application/json": { + schema: { type: "array" }, + example: chains, + } + }, + }, + }, + }, + }) + .addPath("/supply", { + get: { + tags: [TAGS.USAGE], + summary: "ERC20 total supply", + parameters: [ + parameterChain, + parameterString("address"), + parameterLimit, + ], + responses: { + 200: { description: "Array of supply", content: { "application/json": { example: supply_example, schema: { type: "array" } } } }, + 400: { description: "Bad request" }, + }, + }, + }) + .addPath("/contract", { + get: { + tags: [TAGS.USAGE], + summary: "ERC20 contract information", + parameters: [ + parameterChain, + parameterString("address"), + parameterString("symbol"), + parameterString("name"), + parameterLimit, + ], + responses: { + 200: { description: "Array of contracts", content: { "application/json": { example: contract_example, schema: { type: "array" } } } }, + 400: { description: "Bad request" }, + }, + }, + }) + .addPath("/balance", { + get: { + tags: [TAGS.USAGE], + summary: "ERC20 balance changes", + parameters: [ + parameterChain, + parameterString("owner"), + parameterString("contract"), + parameterLimit, + ], + responses: { + 200: { description: "Array of balance changes", content: { "application/json": { example: balance_example, schema: { type: "array" } } } }, + 400: { description: "Bad request" }, + }, + }, + }) + .addPath("/health", { + get: { + tags: [TAGS.HEALTH], + summary: "Performs health checks and checks if the database is accessible", + responses: {200: { description: "OK", content: { "text/plain": {example: "OK"}} } }, + }, + }) + .addPath("/metrics", { + get: { + tags: [TAGS.MONITORING], + summary: "Prometheus metrics", + responses: {200: { description: "Prometheus metrics", content: { "text/plain": { example: await registry.metrics(), schema: { type: "string" } } }}}, + }, + }) + .addPath("/openapi", { + get: { + tags: [TAGS.DOCS], + summary: "OpenAPI specification", + responses: {200: {description: "OpenAPI JSON Specification", content: { "application/json": { schema: { type: "string" } } } }}, + }, + }) + .getSpecAsJson(); \ No newline at end of file diff --git a/src/fetch/supply.ts b/src/fetch/supply.ts new file mode 100644 index 0000000..0a4e224 --- /dev/null +++ b/src/fetch/supply.ts @@ -0,0 +1,20 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/fetch/block.ts +import { makeQuery } from "../clickhouse/makeQuery.js"; +import { logger } from "../logger.js"; +import { getTotalSupply } from "../queries.js"; +import * as prometheus from "../prometheus.js"; +import { toJSON } from "./utils.js"; + +export default async function (req: Request) { + try { + const { searchParams } = new URL(req.url); + logger.info({searchParams: Object.fromEntries(Array.from(searchParams))}); + const query = await getTotalSupply(searchParams); + const response = await makeQuery(query) + return toJSON(response.data); + } catch (e: any) { + logger.error(e); + prometheus.request_error.inc({pathname: "/supply", status: 400}); + return new Response(e.message, { status: 400 }); + } +} \ No newline at end of file diff --git a/src/fetch/utils.ts b/src/fetch/utils.ts new file mode 100644 index 0000000..be854ec --- /dev/null +++ b/src/fetch/utils.ts @@ -0,0 +1,3 @@ +export function toJSON(data: any, status: number = 200) { + return new Response(JSON.stringify(data), { status, headers: { "Content-Type": "application/json" } }); +} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts deleted file mode 100644 index 881085d..0000000 --- a/src/index.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { OpenAPIHono } from "@hono/zod-openapi"; -import { TypedResponse } from "hono"; -import { serveStatic } from "hono/bun"; -import { logger } from "./logger"; -import pkg from "../package.json"; -import * as routes from "./routes"; -import { - type SupplyResponseSchema, - type SupplySchema, - type ContractSchema, - type ContractResponseSchema, - type BalanceSchema, - type BalanceResponseSchema, -} from "./schemas"; -import { getTotalSupply, getContract, getBalance } from "./queries"; -import config from "./config"; -import { HTTPException } from "hono/http-exception"; - - -export function generateApp() { - const app = new OpenAPIHono(); - - app.use("/", serveStatic({ root: "./swagger" })); - - app.doc("/openapi", { - openapi: "3.0.0", - info: { - version: pkg.version, - title: "ERC20 API", - }, - }); - - app.onError((err, c) => { - let error_message = `${err}`; - let error_code = 500; - - if (err instanceof HTTPException) { - error_message = err.message; - error_code = err.status; - } - - logger.error(error_message); - return c.json({ error_message }, error_code); - }); - - - - app.openapi(routes.TotalSupplyQueryRoute, async (c) => { - // @ts-expect-error: Suppress type of parameter expected to be never (see https://github.com/honojs/middleware/issues/200) - const { address, block, contract } = c.req.valid("query") as SupplySchema; - if (contract) { - let supply = await getTotalSupply(address, block); - let contract_info = await getContract(address); - let result = Object.assign({}, supply, contract_info); - return { - response: c.json(result), - } as TypedResponse; - } else { - return { - response: c.json(await getTotalSupply(address, block)), - } as TypedResponse; - } - }); - - app.openapi(routes.ContractQueryRoute, async (c) => { - const { address } = c.req.valid("query") as ContractSchema; - return { - response: c.json(await getContract(address)), - } as TypedResponse; - }); - - app.openapi(routes.BalanceQueryRoute, async (c) => { - // @ts-expect-error: Suppress type of parameter expected to be never (see https://github.com/honojs/middleware/issues/200) - const { wallet, address, block } = c.req.valid("query") as BalanceSchema; - return { - response: c.json(await getBalance(wallet, address, block)), - } as TypedResponse; - }); - - return app; -} -if (config.verbose) logger.enable(); -logger.info( - `Server listening on http://${config.hostname}${config.port}/` -); - -Bun.serve({ - port: config.port, - hostname: config.hostname, - fetch: generateApp().fetch, -}); - diff --git a/src/prometheus.ts b/src/prometheus.ts new file mode 100644 index 0000000..0b8c2a5 --- /dev/null +++ b/src/prometheus.ts @@ -0,0 +1,39 @@ +// From https://github.com/pinax-network/substreams-sink-websockets/blob/main/src/prometheus.ts +import client, { Counter, CounterConfiguration, Gauge, GaugeConfiguration } from 'prom-client'; + +export const registry = new client.Registry(); + +// Metrics +export function registerCounter(name: string, help = "help", labelNames: string[] = [], config?: CounterConfiguration) { + try { + registry.registerMetric(new Counter({ name, help, labelNames, ...config })); + return registry.getSingleMetric(name) as Counter; + } catch (e) { + console.error({name, e}); + throw new Error(`${e}`); + } +} + +export function registerGauge(name: string, help = "help", labelNames: string[] = [], config?: GaugeConfiguration) { + try { + registry.registerMetric(new Gauge({ name, help, labelNames, ...config })); + return registry.getSingleMetric(name) as Gauge; + } catch (e) { + console.error({name, e}); + throw new Error(`${e}`); + } +} + +export async function getSingleMetric(name: string) { + const metric = registry.getSingleMetric(name); + const get = await metric?.get(); + return get?.values[0].value; +} + +// REST API metrics +export const request_error = registerCounter('request_error', 'Total Requests errors', ['pathname', 'status']); +export const request = registerCounter('request', 'Total Requests', ['pathname']); +export const query = registerCounter('query', 'Clickhouse DB queries made'); +export const bytes_read = registerCounter('bytes_read', 'Clickhouse DB Statistics bytes read'); +export const rows_read = registerCounter('rows_read', 'Clickhouse DB Statistics rows read'); +export const elapsed = registerCounter('elapsed', 'Clickhouse DB Statistics query elapsed time'); diff --git a/src/queries.spec.ts b/src/queries.spec.ts new file mode 100644 index 0000000..cb71a3a --- /dev/null +++ b/src/queries.spec.ts @@ -0,0 +1,26 @@ +// from: https://github.com/pinax-network/substreams-clock-api/blob/main/src/queries.spec.ts + +import { expect, test } from "bun:test"; +import { getContracts, getChain, getTotalSupply, getBalanceChanges } from "./queries"; + +const chain = "eth"; +const address = 'dac17f958d2ee523a2206206994597c13d831ec7' + +test("getContracts", () => { + expect(getContracts(new URLSearchParams({ chain, address }))) + .toBe(`SELECT * FROM Contracts JOIN block ON block.block_id = Contracts.block_id WHERE (chain == '${chain}' AND address == '${address}') ORDER BY block_number DESC LIMIT 1`); +}); + +test("getTotalSupply", () => { + expect(getTotalSupply(new URLSearchParams({ chain, address }))) + .toBe(`SELECT * FROM TotalSupply JOIN block ON block.block_id = TotalSupply.block_id WHERE (chain == '${chain}' AND address == '${address}') ORDER BY block_number DESC LIMIT 1`); +}); + +test("getBalanceChanges", () => { + expect(getBalanceChanges(new URLSearchParams({ chain, owner: address }))) + .toBe(`SELECT * FROM balance_changes JOIN block ON block.block_id = balance_changes.block_id WHERE (chain == '${chain}' AND owner == '${address}') ORDER BY block_number DESC LIMIT 1`); +}); + +test("getChain", () => { + expect(getChain()).toBe(`SELECT DISTINCT chain FROM module_hashes`); +}); \ No newline at end of file diff --git a/src/queries.ts b/src/queries.ts index 395acfb..f44b09f 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -1,157 +1,117 @@ -import { ethers } from "ethers"; -import { createClient } from "@clickhouse/client-web"; -import config from "./config"; - -const client = createClient({ - database: config.name, - host: config.dbHost, - username: config.username, - password: config.password, -}); - -function formatAddress(address: string) { - if (address.startsWith("0x")) { - // Remove the "0x" prefix and return the address - return address.slice(2); - } - // If it doesn't start with "0x", return the address as is - return address; -} +import { DEFAULT_SORT_BY } from "./config"; +import { getAddress, parseLimit } from "./utils"; + +export function getTotalSupply(searchParams: URLSearchParams) { + // Params + const address = getAddress(searchParams, "address", false); + const chain = searchParams.get("chain"); + + // Query + const table = 'TotalSupply' + let query = `SELECT * FROM ${table}`; + + // JOIN block table + query += ` JOIN block ON block.block_id = ${table}.block_id`; + + // WHERE statements + const where = []; + + // equals + if (chain) where.push(`chain == '${chain}'`); + if (address) where.push(`address == '${address}'`); + + // TO-DO: sort by timestamp & block number + // https://github.com/pinax-network/substreams-erc20-api/issues/4 -export async function getTotalSupply( - address: string | undefined, - block?: number | undefined -) { - if (address) { - address = formatAddress(address); - if (ethers.isAddress(address)) { - let sqlquery: string = ""; - if (block) { - sqlquery = `SELECT address,supply, block_number AS block,chain - FROM TotalSupply - JOIN block ON block.block_id = TotalSupply.block_id - WHERE address = '${address}' AND block_number >= ${block} - ORDER BY block_number - LIMIT 1`; - } else { - sqlquery = `SELECT address,supply, chain FROM TotalSupply JOIN block ON block.block_id = TotalSupply.block_id WHERE address = '${address}' ORDER BY block_number DESC LIMIT 1`; - } - const resultSet = await client.query({ - query: sqlquery, - format: "JSONEachRow", - }); - const dataset = await resultSet.json(); - if (Array.isArray(dataset) && dataset.length !== 0) return dataset; - else return { error: "Contract data not available" }; - } else { - console.log("Invalid Address"); - return { error: "Invalid Address" }; - } - } + // TO-DO: Filter by symbol & name (INNER JOIN Contracts table) + // https://github.com/pinax-network/substreams-erc20-api/issues/6 + + // Join WHERE statements with AND + if ( where.length ) query += ` WHERE (${where.join(' AND ')})`; + + // Sort and Limit + const limit = parseLimit(searchParams.get("limit")); + const sort_by = searchParams.get("sort_by"); + query += ` ORDER BY block_number ${sort_by ?? DEFAULT_SORT_BY}` + query += ` LIMIT ${limit}` + return query; } -export async function getContract(address: string | undefined) { - if (address) { - address = formatAddress(address); - if (ethers.isAddress(address)) { - let sqlquery: string = `SELECT address,name,symbol,decimals,chain FROM Contracts WHERE address = '${address}'`; - const resultSet = await client.query({ - query: sqlquery, - format: "JSONEachRow", - }); - const dataset = await resultSet.json(); - if (Array.isArray(dataset) && dataset.length !== 0) return dataset; - else return { error: "Contract data not available" }; - } else { - console.log("Invalid Address"); - return { error: "Invalid Address" }; - } - } +export function getContracts(searchParams: URLSearchParams) { + // Params + const chain = searchParams.get("chain"); + const address = getAddress(searchParams, "address", false); + const symbol = searchParams.get("symbol"); + const name = searchParams.get("name"); + + // Query + const table = 'Contracts' + let query = `SELECT * FROM ${table}` + + // JOIN block table + query += ` JOIN block ON block.block_id = ${table}.block_id`; + + // WHERE statements + const where = []; + if ( chain ) where.push(`chain == '${chain}'`); + if ( address ) where.push(`address == '${address}'`); + if ( symbol ) where.push(`symbol == '${symbol}'`); + if ( name ) where.push(`name == '${name}'`); + + // TO-DO: sort by timestamp & block number + // https://github.com/pinax-network/substreams-erc20-api/issues/4 + + // Join WHERE statements with AND + if ( where.length ) query += ` WHERE (${where.join(' AND ')})`; + + // Sort and Limit + const limit = parseLimit(searchParams.get("limit")); + const sort_by = searchParams.get("sort_by"); + query += ` ORDER BY block_number ${sort_by ?? DEFAULT_SORT_BY}` + query += ` LIMIT ${limit}` + return query; } -export async function getBalance( - wallet: string | undefined, - address?: string | undefined, - block?: number | undefined -) { - if (wallet) { - wallet = formatAddress(wallet); - if (address) address = formatAddress(address); - if (ethers.isAddress(wallet)) { - let sqlquery: string = ""; - - //GET all balance of every contract for a wallet LATEST BLOCK - if (!block && !address) { - sqlquery = `WITH RankedBalances AS ( - SELECT - contract, - new_balance AS balance, - chain, - ROW_NUMBER() OVER (PARTITION BY contract ORDER BY block_number DESC) AS rn - FROM balance_changes - JOIN block ON block.block_id = balance_changes.block_id - WHERE owner = '${wallet}' - ) - SELECT contract, balance,chain - FROM RankedBalances - WHERE rn = 1`; - } - - //GET all balance of every contract for a wallet at specific block - else if (block && !address) { - sqlquery = `WITH RankedBalances AS ( - SELECT - contract, - new_balance AS balance, - block_number, - chain, - ROW_NUMBER() OVER (PARTITION BY contract ORDER BY block_number) AS rn - FROM balance_changes - JOIN block ON block.block_id = balance_changes.block_id - WHERE owner = '${wallet}' AND block_number >= ${block} - ) - SELECT contract, balance, block_number,chain - FROM RankedBalances - WHERE rn = 1; - `; - } - //GET balance of a specific contract for a wallet LATEST BLOCK - else if (!block && address) { - if (ethers.isAddress(address)) { - sqlquery = `SELECT contract, new_balance AS balance,chain - FROM balance_changes - JOIN block ON block.block_id = balance_changes.block_id - WHERE owner = '${wallet}' AND contract = '${address}' - ORDER BY block_number DESC - LIMIT 1`; - - console.log(sqlquery); - } else { - console.log("Invalid Address"); - return { error: "Invalid Address" }; - } - } else if (block && address) { - sqlquery = `SELECT contract, - new_balance AS balance, - block_number AS block, - chain - FROM balance_changes - JOIN block ON block.block_id = balance_changes.block_id - WHERE owner = '${wallet}' AND contract = '${address}' AND block_number >= ${block} - ORDER BY block_number - LIMIT 1`; - } - - const resultSet = await client.query({ - query: sqlquery, - format: "JSONEachRow", - }); - const dataset = await resultSet.json(); - if (Array.isArray(dataset) && dataset.length !== 0) return dataset; - else return { error: "Contract data not available" }; - } else { - console.log("Invalid Wallet"); - return { error: "Invalid Wallet" }; - } - } +export function getBalanceChanges(searchParams: URLSearchParams) { + const chain = searchParams.get("chain"); + const contract = getAddress(searchParams, "contract", false); + const owner = getAddress(searchParams, "owner", false); + + // SQL Query + const table = 'balance_changes' + let query = `SELECT * FROM ${table}`; + + // JOIN block table + query += ` JOIN block ON block.block_id = ${table}.block_id`; + + // WHERE statements + const where = []; + + // equals + if ( chain ) where.push(`chain == '${chain}'`); + if ( owner ) where.push(`owner == '${owner}'`); + if ( contract ) where.push(`contract == '${contract}'`); + + // TO-DO: sort by timestamp & block number + // https://github.com/pinax-network/substreams-erc20-api/issues/4 + + // TO-DO: Filter by symbol & name (INNER JOIN Contracts table) + // https://github.com/pinax-network/substreams-erc20-api/issues/6 + + // TO-DO: Filter by transaction_id + // https://github.com/pinax-network/substreams-erc20-api/issues/7 + + // Join WHERE statements with AND + if ( where.length ) query += ` WHERE (${where.join(' AND ')})`; + + // Sort and Limit + const limit = parseLimit(searchParams.get("limit")); + const sort_by = searchParams.get("sort_by"); + query += ` ORDER BY block_number ${sort_by ?? DEFAULT_SORT_BY}` + query += ` LIMIT ${limit}` + return query; } + +export function getChain() { + return `SELECT DISTINCT chain FROM module_hashes`; +} \ No newline at end of file diff --git a/src/routes.ts b/src/routes.ts deleted file mode 100644 index 25854a1..0000000 --- a/src/routes.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { createRoute } from "@hono/zod-openapi"; -import * as schemas from "./schemas"; - - - -export const TotalSupplyQueryRoute = createRoute({ - method: "get", - path: "/supply", - request: { - query: schemas.SupplySchema, - }, - responses: { - 200: { - content: { - "application/json": { - schema: schemas.SupplyResponseSchema, - }, - }, - description: "Get the total supply of an ERC20 contract", - }, - }, -}); - -export const ContractQueryRoute = createRoute({ - method: "get", - path: "/contract", - request: { - query: schemas.ContractSchema, - }, - responses: { - 200: { - content: { - "application/json": { - schema: schemas.ContractResponseSchema, - }, - }, - description: "Get the ERC20 contract information", - }, - }, -}); - -export const BalanceQueryRoute = createRoute({ - method: "get", - path: "/balance", - request: { - query: schemas.BalanceSchema, - }, - responses: { - 200: { - content: { - "application/json": { - schema: schemas.BalanceResponseSchema, - }, - }, - description: "Get the ERC20 contract information", - }, - }, -}); diff --git a/src/schemas.ts b/src/schemas.ts deleted file mode 100644 index 6bed760..0000000 --- a/src/schemas.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { z } from "@hono/zod-openapi"; -import { ethers } from "ethers"; - -export const ContractSchema = z.object({ - address: z - .string() - .refine((val) => ethers.isAddress(val)) - .openapi({ - param: { - name: "address", - in: "query", - }, - example: "dAC17F958D2ee523a2206206994597C13D831ec7", - }), -}); -export type ContractSchema = z.infer; - -export const ContractResponseSchema = z.object({ - address: z.string().openapi({ - example: "dAC17F958D2ee523a2206206994597C13D831ec7", - }), - name: z.string().openapi({ - example: "Tether USD", - }), - symbol: z.string().openapi({ - example: "USDT", - }), - decimals: z.string().or(z.number()).openapi({ - example: "6", - }), - chain: z.string().openapi({ - example: "eth", - }), -}); -export type ContractResponseSchema = z.infer; - -export const SupplySchema = z.object({ - address: z - .string() - .refine((val) => ethers.isAddress(val)) - .openapi({ - param: { - name: "address", - in: "query", - }, - example: "dAC17F958D2ee523a2206206994597C13D831ec7", - }), - block: z.coerce - .number() - .optional() - .openapi({ - param: { - name: "block", - in: "query", - }, - example: 1000000, - }), - - contract: z - .enum(["true", "false"]) - .transform((value) => value === "true") - .optional() - .openapi({ - param: { - name: "contract", - in: "query", - }, - example: true, - }), -}); -export type SupplySchema = z.infer; - -export const SupplyResponseSchema = z.object({ - address: z.string().openapi({ - example: "dAC17F958D2ee523a2206206994597C13D831ec7", - }), - supply: z.string().or(z.number()).openapi({ - example: "10000000", - }), - block: z.number().or(z.string()).openapi({ - example: 1000000, - }), - - chain: z.string().openapi({ - example: "eth", - }), - contract: ContractResponseSchema.optional(), -}); -export type SupplyResponseSchema = z.infer; - -export const BalanceSchema = z.object({ - wallet: z - .string() - .refine((val) => ethers.isAddress(val)) - .openapi({ - param: { - name: "wallet", - in: "query", - }, - example: "a46fcc88d1e03f79e264ec48bcf05094401a6962", - }), - - address: z - .string() - .refine((val) => ethers.isAddress(val)) - .optional() - .openapi({ - param: { - name: "address", - in: "query", - }, - example: "d445d1c4b6d2f048b566ce6c079d20512985854e", - }), - block: z.coerce - .number() - .optional() - .openapi({ - param: { - name: "block", - in: "query", - }, - example: 1000000, - }), -}); -export type BalanceSchema = z.infer; - -export const BalanceResponseSchema = z.object({ - contract: z.string().openapi({ - example: "d445d1c4b6d2f048b566ce6c079d20512985854e", - }), - balance: z.string().or(z.number()).openapi({ - example: "888", - }), - block: z.number().or(z.string()).openapi({ - example: 1009707, - }), - chain: z.string().openapi({ - example: "eth", - }), -}); -export type BalanceResponseSchema = z.infer; diff --git a/src/utils.spec.ts b/src/utils.spec.ts new file mode 100644 index 0000000..86726c8 --- /dev/null +++ b/src/utils.spec.ts @@ -0,0 +1,22 @@ +import { expect, test } from "bun:test"; +import { formatAddress, checkValidAddress, getAddress } from "./utils"; + +const address = "0xdac17f958d2ee523a2206206994597c13d831ec7"; + +test("formatAddress", () => { + expect(formatAddress(address)).toBe("dac17f958d2ee523a2206206994597c13d831ec7") +}); + + +test("checkValidAddress", () => { + checkValidAddress(address) + expect(() => checkValidAddress(address)).not.toThrow(); + expect(() => checkValidAddress("foobar")).toThrow("Invalid address"); +}); + +test("getAddress", () => { + expect(() => getAddress(new URLSearchParams({address: address}), "address", false)).not.toThrow(); + expect(() => getAddress(new URLSearchParams({address: address}), "address", true)).not.toThrow(); + expect(() => getAddress(new URLSearchParams({address: ""}), "address", true)).toThrow("Missing [address] parameter"); + expect(() => getAddress(new URLSearchParams({address: "foobar"}), "address")).toThrow("Invalid address"); +}); diff --git a/src/utils.ts b/src/utils.ts new file mode 100644 index 0000000..44407c9 --- /dev/null +++ b/src/utils.ts @@ -0,0 +1,58 @@ +import { ethers } from "ethers"; +import { config } from "./config"; + +export function getAddress(searchParams: URLSearchParams, key: string, required: boolean = false) { + const address = formatAddress(searchParams.get(key)); + if ( required && !address ) throw new Error(`Missing [${key}] parameter`); + if ( address ) checkValidAddress(address); + return address; +} + +export function formatAddress(address: string|null) { + if ( !address ) return undefined; + if ( address.startsWith("0x") ) { + // Remove the "0x" prefix and return the address + return address.slice(2); + } + // If it doesn't start with "0x", return the address as is + return address; +} + +export function checkValidAddress(address?: string) { + if (!ethers.isAddress(address)) throw new Error("Invalid address"); +} + +export function parseLimit(limit?: string|null|number) { + let value = 1; // default 1 + if (limit) { + if (typeof limit === "string") value = parseInt(limit); + if (typeof limit === "number") value = limit; + } + // limit must be between 1 and maxLimit + if ( value > config.maxLimit ) value = config.maxLimit; + return value; +} + +export function parseBlockId(block_id?: string|null) { + return block_id ? block_id.replace("0x", "") : undefined; +} + +export function parseTimestamp(timestamp?: string|null|number) { + if (timestamp !== undefined && timestamp !== null) { + if (typeof timestamp === "string") { + if (/^[0-9]+$/.test(timestamp)) { + return parseTimestamp(parseInt(timestamp)); + } + // append "Z" to timestamp if it doesn't have it + if (!timestamp.endsWith("Z")) timestamp += "Z"; + return Math.floor(Number(new Date(timestamp)) / 1000); + } + if (typeof timestamp === "number") { + const length = timestamp.toString().length; + if ( length === 10 ) return timestamp; // seconds + if ( length === 13 ) return Math.floor(timestamp / 1000); // convert milliseconds to seconds + throw new Error("Invalid timestamp"); + } + } + return undefined; +} \ No newline at end of file diff --git a/swagger/index.html b/swagger/index.html index e1da840..70037c7 100644 --- a/swagger/index.html +++ b/swagger/index.html @@ -4,7 +4,7 @@ - Substreams Clock API - SwaggerUI + Substreams ERC20 API - SwaggerUI { - const OLD_ENV = process.env; - - afterAll(() => { - process.env = OLD_ENV; - }); - - it('Should load .env variables', () => { - expect(config).toMatchObject(process.env); - }); - - it.skip('Should load default values with no arguments set', () => { - expect(process.argv).toHaveLength(2); // Bun exec and program name - expect(config).toMatchObject({ - port: DEFAULT_PORT, - hostname: DEFAULT_HOSTNAME, - dbHost: DEFAULT_DB_HOST, - name: DEFAULT_DB_NAME, - username: DEFAULT_DB_USERNAME, - password: DEFAULT_DB_PASSWORD, - maxElementsQueried: DEFAULT_MAX_ELEMENTS_QUERIES, - verbose: DEFAULT_VERBOSE - }); - }); -}); \ No newline at end of file diff --git a/test/index.spec.ts b/test/index.spec.ts deleted file mode 100644 index 5135d0b..0000000 --- a/test/index.spec.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { describe, expect, it } from 'bun:test'; -import { generateApp } from '../src/index'; - -const app = generateApp(); - -describe('Index page (/)', () => { - it('Should return 200 Response', async () => { - const res = await app.request('/'); - expect(res.status).toBe(200); - }); -}); - -describe('Supply page (/supply)', () => { - - it('Should return 200 Response for valid address', async () => { - - const validAddress = "cb9df5dc2ed5d7d3972f601acfe35cdbe57341e0" - const res = await app.request('/supply?address=' + validAddress); - - const json = await res.json() as { message: string }; - expect(res.status === 200 || json.message === "Contract data not available").toBe(true); - }); - - it('Should return 200 Response for valid address and block', async () => { - - const validAddress = "cb9df5dc2ed5d7d3972f601acfe35cdbe57341e0" - const res = await app.request('/supply?address=' + validAddress + '&block=1004162'); - - const json = await res.json() as { message: string }; - expect(res.status === 200 || json.message === "Contract data not available").toBe(true); - - }); - - it('Should return 400 Response for invalid address', async () => { - - const validAddress = "awdawd" - const res = await app.request('/supply?address=' + validAddress); - expect(res.status).toBe(400); - }); - - it('Should return 400 Response for valid address but invalid block', async () => { - - const validAddress = "cb9df5dc2ed5d7d3972f601acfe35cdbe57341e0" - const res = await app.request('/supply?address=' + validAddress + '&block=awdawd'); - expect(res.status).toBe(400); - }); -}); - -describe('Contract page (/contract)', () => { - - it('Should return 200 Response for valid address', async () => { - - const validAddress = "cb9df5dc2ed5d7d3972f601acfe35cdbe57341e0" - const res = await app.request('/contract?address=' + validAddress); - const json = await res.json() as { message: string }; - expect(res.status === 200 || json.message === "Contract data not available").toBe(true); - }); - - it('Should return 400 Response for invalid address', async () => { - - const validAddress = "awdawd" - const res = await app.request('/contract?address=' + validAddress); - expect(res.status).toBe(400); - }); -}); - - -describe('Balance page (/balance)', () => { - - it('Should return 200 Response for valid address', async () => { - - const validWallet = "39fA8c5f2793459D6622857E7D9FbB4BD91766d3" - const res = await app.request('/balance?wallet=' + validWallet); - const json = await res.json() as { message: string }; - expect(res.status === 200 || json.message === "Contract data not available").toBe(true); - }); - - it('Should return 200 Response for valid wallet and block', async () => { - - const validWallet = "39fA8c5f2793459D6622857E7D9FbB4BD91766d3" - const res = await app.request('/balance?wallet=' + validWallet + '&block=1000000'); - const json = await res.json() as { message: string }; - expect(res.status === 200 || json.message === "Contract data not available").toBe(true); - - }); - - - it('Should return 200 Response for valid wallet and valid address', async () => { - - const validWallet = "39fA8c5f2793459D6622857E7D9FbB4BD91766d3" - const res = await app.request('/balance?wallet=' + validWallet + '&address=c083e9947Cf02b8FfC7D3090AE9AEA72DF98FD47'); - const json = await res.json() as { message: string }; - expect(res.status === 200 || json.message === "Contract data not available").toBe(true); - }); - - it('Should return 400 Response for valid wallet and invalid address', async () => { - - const validWallet = "39fA8c5f2793459D6622857E7D9FbB4BD91766d3" - const res = await app.request('/balance?address=' + validWallet + '&address=3rrw3r'); - expect(res.status).toBe(400); - }); - - it('Should return 400 Response for invalid wallet', async () => { - - const validWallet = "awdawdaw" - const res = await app.request('/balance?wallet=' + validWallet); - expect(res.status).toBe(400); - }); - - it('Should return 400 Response for valid wallet but invalid block', async () => { - - const validWallet = "39fA8c5f2793459D6622857E7D9FbB4BD91766d3" - const res = await app.request('/balance?wallet=' + validWallet + '&block=awdawd'); - expect(res.status).toBe(400); - }); -}); \ No newline at end of file