Updated signing process

bin/cli.ts

@@ -4,27 +4,29 @@ import { Option } from "commander";
 import { commander, logger } from "substreams-sink";
 import { action } from "../index.js";
 import pkg from "../package.json" assert { type: "json" };
+import { AuthMode, authModes } from "../src/auth/index.js";
+import { keyPair } from "../src/auth/utils.js";
 import { ping } from "../src/ping.js";
-import { keyPair } from "../src/signMessage.js";
 
 export interface WebhookRunOptions extends commander.RunOptions {
   webhookUrl: string;
   secretKey: string;
   disablePing: boolean;
+  signature: AuthMode;
+  expiryTime: number;
 }
 
+const signatureOption = new Option("--signature <string>", "Signature mode to use when sending data payloads").choices(authModes).env("SIGNATURE").default("payload")
+const expirationOption = new Option("--expiry-time <number>", "Time before a transmission becomes invalid (in seconds)").env("EXPIRY_TIME").default(40)
+
 // Run Webhook Sink
 const program = commander.program(pkg);
 const command = commander.run(program, pkg);
-command.addOption(
-  new Option("--webhook-url <string>", "Webhook URL to send POST").makeOptionMandatory().env("WEBHOOK_URL"),
-);
-command.addOption(
-  new Option("--secret-key <string>", "TweetNaCl Secret-key to sign POST data payload")
-    .makeOptionMandatory()
-    .env("SECRET_KEY"),
-);
+command.addOption(new Option("--webhook-url <string>", "Webhook URL to send POST").makeOptionMandatory().env("WEBHOOK_URL"));
+command.addOption(new Option("--secret-key <string>", "TweetNaCl Secret-key to sign POST data payload").makeOptionMandatory().env("SECRET_KEY"));
 command.addOption(new Option("--disable-ping", "Disable ping on init").env("DISABLE_PING").default(false));
+command.addOption(signatureOption);
+command.addOption(expirationOption);
 command.action(action);
 
 program
@@ -40,14 +42,13 @@ program
   .command("ping")
   .description("Ping Webhook URL")
   .addOption(new Option("--webhook-url <string>", "Webhook URL to send POST").makeOptionMandatory().env("WEBHOOK_URL"))
-  .addOption(
-    new Option("--secret-key <string>", "TweetNaCl Secret-key to sign POST data payload")
-      .makeOptionMandatory()
-      .env("SECRET_KEY"),
-  )
-  .action(async (options) => {
+  .addOption(new Option("--secret-key <string>", "TweetNaCl Secret-key to sign POST data payload").makeOptionMandatory().env("SECRET_KEY"))
+  .addOption(signatureOption)
+  .addOption(expirationOption)
+  .action(async (options: WebhookRunOptions) => {
     logger.settings.type = "hidden";
-    const response = await ping(options.webhookUrl, options.secretKey);
+    const signerOptions = { secretKey: options.secretKey, expirationTime: options.expiryTime };
+    const response = await ping(options.webhookUrl, options.signature, signerOptions);
     if (response) console.log("✅ OK");
     else console.log("⁉️ ERROR");
   });

index.ts

@@ -1,10 +1,10 @@
 import PQueue from "p-queue";
 import { http, logger, setup } from "substreams-sink";
 import { postWebhook } from "./src/postWebhook.js";
-import { signMessage } from "./src/signMessage.js";
 
 import type { SessionInit } from "@substreams/core/proto";
 import type { WebhookRunOptions } from "./bin/cli.js";
+import { makeSigner } from "./src/auth/index.js";
 import { banner } from "./src/banner.js";
 import { toText } from "./src/http.js";
 import { ping } from "./src/ping.js";
@@ -16,9 +16,13 @@ export async function action(options: WebhookRunOptions) {
   // Queue
   const queue = new PQueue({ concurrency: 1 }); // all messages are sent in block order, no need to parallelize
 
+  // Signer
+  const signerOptions = { secretKey: options.secretKey, expirationTime: options.expiryTime };
+  const signer = makeSigner(options.signature, signerOptions);
+
   // Ping URL to check if it's valid
   if (!options.disablePing) {
-    if (!(await ping(options.webhookUrl, options.secretKey))) {
+    if (!(await ping(options.webhookUrl, options.signature, signerOptions))) {
       logger.error("exiting from invalid PING response");
       process.exit(1);
     }
@@ -53,14 +57,12 @@ export async function action(options: WebhookRunOptions) {
         moduleHash,
       },
     };
-    // Sign body
-    const seconds = Number(clock.timestamp.seconds);
+
     const body = JSON.stringify({ ...metadata, data });
-    const signature = signMessage(seconds, body, options.secretKey);
 
     // Queue POST
     queue.add(async () => {
-      const response = await postWebhook(options.webhookUrl, body, signature, seconds);
+      const response = await postWebhook(options.webhookUrl, metadata.clock.number, body, signer);
       logger.info("POST", response, metadata);
     });
   });

package.json

@@ -19,7 +19,7 @@
     "test": "bun test",
     "prepublishOnly": "tsc",
     "build": "bun build --compile ./index.ts --outfile substreams-sink-webhook",
-    "bench": "bun ./src/*.bench.ts"
+    "bench": "bun ./src/**/*.bench.ts"
   },
   "dependencies": {
     "p-queue": "latest",

src/auth/auth.bench.ts

@@ -0,0 +1,36 @@
+import { bench, group, run } from "mitata";
+import { makeSigner, makeVerifier } from "./index.js";
+
+const secretKey =
+  "3faae992336ea6599fbee55bb2605f1a1297c7288b860725cdfc8794413559dba3cb7366ee8ca77225b4d41772e270e4e831d171d1de71d91707c42e7ba82cc9";
+const publicKey = "a3cb7366ee8ca77225b4d41772e270e4e831d171d1de71d91707c42e7ba82cc9";
+
+const cacheSignature =
+  "0mKZAisTwl5IiRkc229KuPowpSS8pEsXQr7e6rsUXKmXkLoJRn8TZfhwruEjbshoLNw2kO2kyCZs_1EkR9cnC3siZXhwIjoyNTI0NjA4MDAwMDAwLCJpZCI6ImEzY2I3MzY2ZWU4Y2E3NzIyNWI0ZDQxNzcyZTI3MGU0ZTgzMWQxNzFkMWRlNzFkOTE3MDdjNDJlN2JhODJjYzkifQ";
+const payloadSignature = "c3233e06e21bae7dce5428091abbcea94c75e4a0b6b5a52a435f36053d178ca6aac58c507f8bba9aaccd3fe175dad1a4a0590ad18763662b735888338ebe0509";
+
+const body =
+  '{"cursor":"gBCLb0z81lU8vbvZVzJkEaWwLpc_DFhqVQ3jLxVJgYH2pSTFicymUzd9bx2GlKH51RboGgmo19eZRX588ZED7YW8y7FhuSM6EHh4wNzo87Dne6KjPQlIIOhjC-iJMNncUT7SYgz9f7UI5N_nb6XZMxMyMZEuK2blizdZqoZXIfAVsHthkjz6cJ6Bga_A-YtEq-AnEuf1xn6lDzF1Lx4LOc_RNqGe6z4nN3Rq","clock":{"timestamp":"2023-06-15T04:21:58.000Z","number":250665484,"id":"0ef0da0cf870f489833ac498da073acadf895d22f3dce68483aa43cac1d27b17"},"manifest":{"chain":"wax","moduleName":"map_transfers","moduleHash":"6aa24e6aa34db4a4faf55c69c6f612aeb06053c2"},"data":{"items":[{"trxId":"dd93c64db8ff91cfac74e731fd518548aa831be3d833e6a1fefeac69d2ddd138","actionOrdinal":2,"contract":"eosio.token","action":"transfer","symcode":"WAX","from":"banxawallet1","to":"atomicmarket","quantity":"1340.00000000 WAX","memo":"deposit","precision":8,"amount":"134000000000","value":1340},{"trxId":"dd93c64db8ff91cfac74e731fd518548aa831be3d833e6a1fefeac69d2ddd138","actionOrdinal":7,"contract":"eosio.token","action":"transfer","symcode":"WAX","from":"atomicmarket","to":"jft4m.c.wam","quantity":"1206.00000000 WAX","memo":"AtomicMarket Sale Payout - ID #129675349","precision":8,"amount":"120600000000","value":1206}]}}';
+
+const payloadSigner = makeSigner("payload", { secretKey, expirationTime: 0 });
+const payloadVerifier = makeVerifier("payload", [publicKey]);
+
+const cacheSigner = makeSigner("cached", { secretKey, expirationTime: 60 });
+const cacheVerifier = makeVerifier("cached", [publicKey]);
+
+
+group("sign", () => {
+  // @ts-expect-error
+  bench("[--signature cached] sign - cache disabled", () => cacheSigner.refreshSignature());
+  bench("[--signature cached] sign - cache enabled", () => cacheSigner.signature(0, body));
+  bench("[--signature payload] sign", () => payloadSigner.signature(0, body));
+});
+
+group("verify", () => {
+  // @ts-expect-error
+  bench("[--signature cached] verify - cache disabled", () => cacheVerifier.verifyMessage(cacheSignature));
+  bench("[--signature cached] verify - cache enabled", () => cacheVerifier.verify(cacheSignature, ""));
+  bench("[--signature payload] verify", () => payloadVerifier.verify(payloadSignature, body));
+});
+
+await run({ avg: true, json: false, colors: true, min_max: true, collect: false, percentiles: false });

src/auth/cached-signer.test.ts

@@ -0,0 +1,51 @@
+import { beforeEach, describe, expect, setSystemTime, spyOn, test } from "bun:test";
+import { makeSigner, makeVerifier } from "./index.js";
+
+const secretKey =
+  "3faae992336ea6599fbee55bb2605f1a1297c7288b860725cdfc8794413559dba3cb7366ee8ca77225b4d41772e270e4e831d171d1de71d91707c42e7ba82cc9";
+const publicKey = "a3cb7366ee8ca77225b4d41772e270e4e831d171d1de71d91707c42e7ba82cc9";
+
+describe("CachedSigner", () => {
+  beforeEach(() => {
+    // Reset clock
+    setSystemTime();
+  });
+
+  // This test will be invalid from January 1, 2050
+  test("sign", () => {
+    // Make the token expire in 2050 by modifying the current time
+    setSystemTime(new Date(2050, 0, 1));
+    const signer = makeSigner("cached", { secretKey, expirationTime: 0 });
+    // Reset time to regular clock
+    setSystemTime();
+    const verifier = makeVerifier("cached",  [publicKey])
+
+    const expectedSignature =
+      "0mKZAisTwl5IiRkc229KuPowpSS8pEsXQr7e6rsUXKmXkLoJRn8TZfhwruEjbshoLNw2kO2kyCZs_1EkR9cnC3siZXhwIjoyNTI0NjA4MDAwMDAwLCJpZCI6ImEzY2I3MzY2ZWU4Y2E3NzIyNWI0ZDQxNzcyZTI3MGU0ZTgzMWQxNzFkMWRlNzFkOTE3MDdjNDJlN2JhODJjYzkifQ";
+
+      const signature = signer.signature(0, "")
+    expect(signature).toBe(expectedSignature);
+    expect(verifier.verify(signature, "")).toBeTrue();
+  });
+
+  test("sign cache", () => {
+    setSystemTime(new Date("2000-01-01T00:00:00.000Z"));
+
+    const signer = makeSigner("cached", { secretKey, expirationTime: 60 });
+    const refreshSignatureSpy = spyOn(signer as any, "refreshSignature");
+
+    // A signature is automatically generated when the Signer object is created
+    signer.signature(0, "");
+    expect(refreshSignatureSpy).toHaveBeenCalledTimes(0);
+
+    // Requesting the signature in the first 60% of the time window will not regenerate it (0.6*60s = 36s)
+    setSystemTime(new Date("2000-01-01T00:00:35.000Z"));
+    signer.signature(0, "");
+    expect(refreshSignatureSpy).toHaveBeenCalledTimes(0);
+
+    // Requesting the signature after the first 60% of the time window will regenerate it
+    setSystemTime(new Date("2000-01-01T00:00:36.000Z"));
+    signer.signature(0, "");
+    expect(refreshSignatureSpy).toHaveBeenCalledTimes(1);
+  });
+});

src/auth/cached-signer.ts

@@ -0,0 +1,36 @@
+import nacl from "tweetnacl";
+import { Signer } from "./index.js";
+
+export class CachedSigner implements Signer {
+  // After how much time (in % of expiration time) should a token be regenerated.
+  private readonly GRACE_PERIOD = 0.6;
+
+  private publicKey: string;
+  private latestSignature!: { signature: string; expiryTime: number };
+
+  constructor(private secretKey: string, private expirationTime: number) {
+    this.publicKey = this.secretKey.substring(nacl.sign.secretKeyLength);
+    this.refreshSignature();
+  }
+
+  public signature(_: number, __: string) {
+    if (this.latestSignature.expiryTime - this.now() <= (1 - this.GRACE_PERIOD) * this.expirationTime * 1000) {
+      this.refreshSignature();
+    }
+
+    return this.latestSignature.signature;
+  }
+
+  private refreshSignature() {
+    const expiryTime = this.now() + this.expirationTime * 1000;
+    const payload = JSON.stringify({ exp: expiryTime, id: this.publicKey });
+    const signedBuffer = nacl.sign(Buffer.from(payload), Buffer.from(this.secretKey, "hex"));
+
+    const signature = Buffer.from(signedBuffer).toString("base64url");
+    this.latestSignature = { expiryTime, signature };
+  }
+
+  private now() {
+    return new Date().getTime();
+  }
+}

src/auth/cached-verifier.test.ts

@@ -0,0 +1,58 @@
+import { expect, setSystemTime, spyOn, test } from "bun:test";
+import { makeSigner, makeVerifier } from "./index.js";
+
+const secretKey =
+  "3faae992336ea6599fbee55bb2605f1a1297c7288b860725cdfc8794413559dba3cb7366ee8ca77225b4d41772e270e4e831d171d1de71d91707c42e7ba82cc9";
+const publicKey = "a3cb7366ee8ca77225b4d41772e270e4e831d171d1de71d91707c42e7ba82cc9";
+
+// This test will be invalid from January 1, 2050
+test("verify", () => {
+  const invalidPublicKey = "36657c7498f2ff2e9a520dcfbdad4e7c1e5354a75623165e28f6577a45a9eec3";
+
+  const expiry = new Date(2050, 0, 1);
+  const expired = new Date(2000, 0, 1);
+
+  const tests = [
+    { key: publicKey, expiry: expiry, expected: true },
+    { key: publicKey, expiry: expired, expected: false },
+    { key: invalidPublicKey, expiry: expired, expected: false },
+    { key: invalidPublicKey, expiry: expired, expected: false },
+  ];
+
+  for (const test of tests) {
+    setSystemTime(test.expiry);
+    const signer = makeSigner("cached", { secretKey, expirationTime: 0 });
+    setSystemTime();
+    const verifier = makeVerifier("cached", [test.key]);
+
+    expect(verifier.verify(signer.signature(0, ""), "")).toBe(test.expected);
+  }
+});
+
+test("verify cache", () => {
+  setSystemTime(new Date("2000-01-01T00:00:00.000Z"));
+  const signer = makeSigner("cached", { secretKey, expirationTime: 60 });
+  const signature = signer.signature(0, "");
+
+  const verifier = makeVerifier("cached", [publicKey]);
+  const verifyMessageSpy = spyOn(verifier as any, "verifyMessage");
+
+  expect(verifier.verify(signature, "")).toBeTrue();
+  expect(verifyMessageSpy).toHaveBeenCalledTimes(1);
+
+  // This signature is already known, we do not need to revalidate it
+  expect(verifier.verify(signature, "")).toBeTrue();
+  expect(verifyMessageSpy).toHaveBeenCalledTimes(1);
+
+  // This signature expires in 1s, but it is still valid. We do not need to revalide it.
+  setSystemTime(new Date("2000-01-01T00:00:59.000Z"));
+  expect(verifier.verify(signature, "")).toBeTrue();
+  expect(verifyMessageSpy).toHaveBeenCalledTimes(1);
+
+  // This signature is expired, it should be removed from the cache.
+  setSystemTime(new Date("2000-01-01T00:01:00.000Z"));
+  expect(verifier.verify(signature, "")).toBeFalse();
+  // @ts-expect-error
+  expect(verifier.expirationTimes).toEqual({});
+  expect(verifyMessageSpy).toHaveBeenCalledTimes(1);
+});

src/auth/cached-verifier.ts

@@ -0,0 +1,63 @@
+import { logger } from "substreams-sink";
+import nacl from "tweetnacl";
+import { Verifier } from "./index.js";
+
+export class CachedVerifier implements Verifier {
+  private expirationTimes: { [message: string]: number } = {};
+
+  constructor(public knownPublicKeys: string[]) {}
+
+  public verify(signature: string): boolean {
+    if (!this.expirationTimes[signature]) {
+      this.removedExpired();
+      return this.verifyMessage(signature);
+    }
+
+    if (this.now() >= this.expirationTimes[signature]) {
+      this.removedExpired();
+      return false;
+    }
+
+    return true;
+  }
+
+  private verifyMessage(message: string) {
+    try {
+      const msg = Buffer.from(message, "base64url");
+      const signature = msg.subarray(0, nacl.sign.signatureLength);
+      const payloadBuffer = msg.subarray(nacl.sign.signatureLength);
+      const payload = JSON.parse(payloadBuffer.toString("utf-8"));
+
+      if (this.now() >= payload.exp) {
+        throw new Error("signature has expired");
+      }
+
+      const publicKey = this.knownPublicKeys.find((key) => key === payload.id);
+      if (!publicKey) {
+        throw new Error("unknown public key");
+      }
+
+      if (!nacl.sign.detached.verify(payloadBuffer, signature, Buffer.from(publicKey, "hex"))) {
+        throw new Error("invalid signature");
+      }
+
+      this.expirationTimes[message] = payload.exp;
+      return true;
+    } catch (err) {
+      logger.error(err);
+      return false;
+    }
+  }
+
+  private removedExpired() {
+    for (const key of Object.keys(this.expirationTimes)) {
+      if (this.now() >= this.expirationTimes[key]) {
+        delete this.expirationTimes[key];
+      }
+    }
+  }
+
+  private now() {
+    return new Date().getTime();
+  }
+}