From f988a2c074355981188ffca942d43e126526441c Mon Sep 17 00:00:00 2001 From: NathanosDev Date: Mon, 29 Apr 2024 16:25:32 +0200 Subject: [PATCH 1/2] feat: add support for proof of absence in certificate lookups --- packages/agent/src/agent/http/index.ts | 13 +- packages/agent/src/canisterStatus/index.ts | 25 +- packages/agent/src/certificate.test.ts | 304 +++++++++++++++++---- packages/agent/src/certificate.ts | 293 +++++++++++++++----- 4 files changed, 505 insertions(+), 130 deletions(-) diff --git a/packages/agent/src/agent/http/index.ts b/packages/agent/src/agent/http/index.ts index 970f21cc..28f28728 100644 --- a/packages/agent/src/agent/http/index.ts +++ b/packages/agent/src/agent/http/index.ts @@ -29,7 +29,12 @@ import { } from './types'; import { AgentHTTPResponseError } from './errors'; import { SubnetStatus, request } from '../../canisterStatus'; -import { CertificateVerificationError, HashTree, lookup_path } from '../../certificate'; +import { + CertificateVerificationError, + HashTree, + LookupStatus, + lookup_path, +} from '../../certificate'; import { ed25519 } from '@noble/curves/ed25519'; import { ExpirableMap } from '../../utils/expirableMap'; import { Ed25519PublicKey } from '../../public_key'; @@ -902,14 +907,14 @@ export class HttpAgent implements Agent { throw new Error('Could not decode time from response'); } const timeLookup = lookup_path(['time'], tree); - if (!timeLookup) { + if (timeLookup.status !== LookupStatus.Found) { throw new Error('Time was not found in the response or was not in its expected format.'); } - if (!(timeLookup instanceof ArrayBuffer) && !ArrayBuffer.isView(timeLookup)) { + if (!(timeLookup.value instanceof ArrayBuffer) && !ArrayBuffer.isView(timeLookup)) { throw new Error('Time was not found in the response or was not in its expected format.'); } - const date = decodeTime(bufFromBufLike(timeLookup)); + const date = decodeTime(bufFromBufLike(timeLookup.value as ArrayBuffer)); this.log('Time from response:', date); this.log('Time from response in milliseconds:', Number(date)); return Number(date); diff --git a/packages/agent/src/canisterStatus/index.ts b/packages/agent/src/canisterStatus/index.ts index dd7bc579..2bed3789 100644 --- a/packages/agent/src/canisterStatus/index.ts +++ b/packages/agent/src/canisterStatus/index.ts @@ -9,7 +9,7 @@ import { HashTree, flatten_forks, check_canister_ranges, - lookupResultToBuffer, + LookupStatus, lookup_path, } from '../certificate'; import { toHex } from '../utils/buffer'; @@ -160,7 +160,7 @@ export const request = async (options: { } else { return { path: path, - data: lookupResultToBuffer(cert.lookup(encodePath(path, canisterId))), + data: cert.lookup(encodePath(path, canisterId)), }; } }; @@ -290,14 +290,25 @@ export const fetchNodeKeys = ( throw new Error('Canister not in range'); } - const nodeTree = lookup_path(['subnet', delegation?.subnet_id as ArrayBuffer, 'node'], tree); - const nodeForks = flatten_forks(nodeTree as HashTree) as HashTree[]; - nodeForks.length; + const subnetLookupResult = lookup_path(['subnet', delegation.subnet_id, 'node'], tree); + if (subnetLookupResult.status !== LookupStatus.Found) { + throw new Error('Node not found'); + } + if (subnetLookupResult.value instanceof ArrayBuffer) { + throw new Error('Invalid node tree'); + } + + const nodeForks = flatten_forks(subnetLookupResult.value); const nodeKeys = new Map(); + nodeForks.forEach(fork => { - Object.getPrototypeOf(new Uint8Array(fork[1] as ArrayBuffer)); const node_id = Principal.from(new Uint8Array(fork[1] as ArrayBuffer)).toText(); - const derEncodedPublicKey = lookup_path(['public_key'], fork[2] as HashTree) as ArrayBuffer; + const publicKeyLookupResult = lookup_path(['public_key'], fork[2] as HashTree); + if (publicKeyLookupResult.status !== LookupStatus.Found) { + throw new Error('Public key not found'); + } + + const derEncodedPublicKey = publicKeyLookupResult.value as ArrayBuffer; if (derEncodedPublicKey.byteLength !== 44) { throw new Error('Invalid public key length'); } else { diff --git a/packages/agent/src/certificate.test.ts b/packages/agent/src/certificate.test.ts index fe22fe79..7e925f37 100644 --- a/packages/agent/src/certificate.test.ts +++ b/packages/agent/src/certificate.test.ts @@ -8,7 +8,6 @@ import * as Cert from './certificate'; import { fromHex, toHex } from './utils/buffer'; import { Principal } from '@dfinity/principal'; import { decodeTime } from './utils/leb'; -import { lookupResultToBuffer, lookup_path } from './certificate'; import { readFileSync } from 'fs'; import path from 'path'; @@ -20,6 +19,24 @@ function pruned(str: string): ArrayBuffer { return fromHex(str); } +function bufferEqualityTester(a: unknown, b: unknown): boolean | undefined { + if (a instanceof ArrayBuffer && b instanceof ArrayBuffer) { + return Cert.isBufferEqual(a, b); + } + + if (a instanceof ArrayBuffer && b instanceof Uint8Array) { + return Cert.isBufferEqual(a, b.buffer); + } + + if (a instanceof Uint8Array && b instanceof ArrayBuffer) { + return Cert.isBufferEqual(a.buffer, b); + } + + return undefined; +} + +(expect as any).addEqualityTesters([bufferEqualityTester]); + // Root public key for the IC main net, encoded as hex const IC_ROOT_KEY = '308182301d060d2b0601040182dc7c0503010201060c2b0601040182dc7c05030201036100814' + @@ -33,17 +50,29 @@ test('hash tree', async () => { '83024162820344676f6f648301830241638100830241648203476d6f726e696e67', ); const expected: Cert.HashTree = [ - 1, + Cert.NodeType.Fork, [ - 1, + Cert.NodeType.Fork, [ - 2, + Cert.NodeType.Labeled, label('a'), - [1, [1, [2, label('x'), [3, label('hello')]], [0]], [2, label('y'), [3, label('world')]]], + [ + Cert.NodeType.Fork, + [ + Cert.NodeType.Fork, + [Cert.NodeType.Labeled, label('x'), [3, label('hello')]], + [Cert.NodeType.Empty], + ], + [Cert.NodeType.Labeled, label('y'), [Cert.NodeType.Leaf, label('world')]], + ], ], - [2, label('b'), [3, label('good')]], + [Cert.NodeType.Labeled, label('b'), [Cert.NodeType.Leaf, label('good')]], + ], + [ + Cert.NodeType.Fork, + [Cert.NodeType.Labeled, label('c'), [Cert.NodeType.Empty]], + [Cert.NodeType.Labeled, label('d'), [Cert.NodeType.Leaf, label('morning')]], ], - [1, [2, label('c'), [0]], [2, label('d'), [3, label('morning')]]], ]; const tree: Cert.HashTree = cbor.decode(new Uint8Array(cborEncode)); expect(tree).toMatchObject(expected); @@ -61,28 +90,34 @@ test('pruned hash tree', async () => { 'bd2e806edba78006479c9877fed4eb464a25485465af601d830241648203476d6f726e696e67', ); const expected: Cert.HashTree = [ - 1, + Cert.NodeType.Fork, [ - 1, + Cert.NodeType.Fork, [ - 2, + Cert.NodeType.Labeled, label('a'), [ - 1, + Cert.NodeType.Fork, [4, pruned('1b4feff9bef8131788b0c9dc6dbad6e81e524249c879e9f10f71ce3749f5a638')], - [2, label('y'), [3, label('world')]], + [Cert.NodeType.Labeled, label('y'), [Cert.NodeType.Leaf, label('world')]], ], ], [ - 2, + Cert.NodeType.Labeled, label('b'), - [4, pruned('7b32ac0c6ba8ce35ac82c255fc7906f7fc130dab2a090f80fe12f9c2cae83ba6')], + [ + Cert.NodeType.Pruned, + pruned('7b32ac0c6ba8ce35ac82c255fc7906f7fc130dab2a090f80fe12f9c2cae83ba6'), + ], ], ], [ - 1, - [4, pruned('ec8324b8a1f1ac16bd2e806edba78006479c9877fed4eb464a25485465af601d')], - [2, label('d'), [3, label('morning')]], + Cert.NodeType.Fork, + [ + Cert.NodeType.Pruned, + pruned('ec8324b8a1f1ac16bd2e806edba78006479c9877fed4eb464a25485465af601d'), + ], + [Cert.NodeType.Labeled, label('d'), [Cert.NodeType.Leaf, label('morning')]], ], ]; const tree: Cert.HashTree = cbor.decode(new Uint8Array(cborEncode)); @@ -92,51 +127,220 @@ test('pruned hash tree', async () => { ); }); -test('lookup', () => { +describe('lookup', () => { const tree: Cert.HashTree = [ - 1, + Cert.NodeType.Fork, [ - 1, + Cert.NodeType.Fork, [ - 2, - label('a'), + Cert.NodeType.Fork, [ - 1, - [4, pruned('1b4feff9bef8131788b0c9dc6dbad6e81e524249c879e9f10f71ce3749f5a638')], - [2, label('y'), [3, label('world')]], + Cert.NodeType.Labeled, + label('a'), + [ + Cert.NodeType.Pruned, + pruned('1b842dfc254abb83e61bcdd7b7c24492322a2e1b006e6d20b88bedd147c248fc'), + ], ], + [Cert.NodeType.Labeled, label('c'), [Cert.NodeType.Leaf, label('hello')]], ], [ - 2, - label('b'), - [4, pruned('7b32ac0c6ba8ce35ac82c255fc7906f7fc130dab2a090f80fe12f9c2cae83ba6')], + Cert.NodeType.Labeled, + label('d'), + [ + Cert.NodeType.Fork, + [Cert.NodeType.Labeled, label('1'), [Cert.NodeType.Leaf, label('42')]], + [ + Cert.NodeType.Pruned, + pruned('5ec92bd71f697eee773919200a9718c4719495a4c6bba52acc408bd79b4bf57f'), + ], + ], ], ], [ - 1, - [4, pruned('ec8324b8a1f1ac16bd2e806edba78006479c9877fed4eb464a25485465af601d')], - [2, label('d'), [3, label('morning')]], + Cert.NodeType.Fork, + [Cert.NodeType.Labeled, label('e'), [Cert.NodeType.Leaf, label('world')]], + [Cert.NodeType.Labeled, label('g'), [Cert.NodeType.Empty]], ], ]; - function toText(buff: ArrayBuffer): string { - const decoder = new TextDecoder(); - const t = decoder.decode(buff); - return t; - } - function fromText(str: string): ArrayBuffer { - return new TextEncoder().encode(str); - } - expect(Cert.lookup_path([fromText('a'), fromText('a')], tree)).toEqual(undefined); - expect( - toText(lookupResultToBuffer(Cert.lookup_path([fromText('a'), fromText('y')], tree))!), - ).toEqual('world'); - expect(Cert.lookup_path([fromText('aa')], tree)).toEqual(undefined); - expect(Cert.lookup_path([fromText('ax')], tree)).toEqual(undefined); - expect(Cert.lookup_path([fromText('b')], tree)).toEqual([4, new ArrayBuffer(0)]); - expect(Cert.lookup_path([fromText('bb')], tree)).toEqual(undefined); - expect(toText(lookupResultToBuffer(Cert.lookup_path([fromText('d')], tree))!)).toEqual('morning'); - expect(Cert.lookup_path([fromText('e')], tree)).toEqual(undefined); + test('subtree_a', () => { + // a subtree at label `a` exists + const lookup_a = Cert.find_label(label('a'), tree); + expect(lookup_a).toEqual({ + status: Cert.LookupStatus.Found, + value: [ + Cert.NodeType.Pruned, + pruned('1b842dfc254abb83e61bcdd7b7c24492322a2e1b006e6d20b88bedd147c248fc'), + ], + }); + expect(Cert.lookup_path([label('a')], tree)).toEqual({ + status: Cert.LookupStatus.Found, + value: [ + Cert.NodeType.Pruned, + pruned('1b842dfc254abb83e61bcdd7b7c24492322a2e1b006e6d20b88bedd147c248fc'), + ], + }); + + // the subtree at label `a` is pruned, + // so any nested lookups should return Unknown + const tree_a = (lookup_a as Cert.LookupResultFound).value as Cert.HashTree; + + expect(Cert.find_label(label('1'), tree_a)).toEqual({ + status: Cert.LookupStatus.Unknown, + }); + expect(Cert.lookup_path([label('1')], tree_a)).toEqual({ + status: Cert.LookupStatus.Unknown, + }); + expect(Cert.lookup_path([label('a'), label('1')], tree)).toEqual({ + status: Cert.LookupStatus.Unknown, + }); + + expect(Cert.find_label(label('2'), tree_a)).toEqual({ + status: Cert.LookupStatus.Unknown, + }); + expect(Cert.lookup_path([label('2')], tree_a)).toEqual({ + status: Cert.LookupStatus.Unknown, + }); + expect(Cert.lookup_path([label('a'), label('2')], tree)).toEqual({ + status: Cert.LookupStatus.Unknown, + }); + }); + + test('subtree_b', () => { + // there are no nodes between labels `a` and `c`, + // so the subtree at label `b` is provably Absent + expect(Cert.find_label(label('b'), tree)).toEqual({ + status: Cert.LookupStatus.Absent, + }); + expect(Cert.lookup_path([label('b')], tree)).toEqual({ + status: Cert.LookupStatus.Absent, + }); + }); + + test('subtree_c', () => { + // a subtree at label `c` exists + expect(Cert.find_label(label('c'), tree)).toEqual({ + status: Cert.LookupStatus.Found, + value: [Cert.NodeType.Leaf, label('hello')], + }); + expect(Cert.lookup_path([label('c')], tree)).toEqual({ + status: Cert.LookupStatus.Found, + value: label('hello'), + }); + }); + + test('subtree_d', () => { + // a subtree at label `d` exists + const lookup_d = Cert.find_label(label('d'), tree); + expect(lookup_d).toEqual({ + status: Cert.LookupStatus.Found, + value: [ + Cert.NodeType.Fork, + [Cert.NodeType.Labeled, label('1'), [Cert.NodeType.Leaf, label('42')]], + [ + Cert.NodeType.Pruned, + pruned('5ec92bd71f697eee773919200a9718c4719495a4c6bba52acc408bd79b4bf57f'), + ], + ], + }); + expect(Cert.lookup_path([label('d')], tree)).toEqual({ + status: Cert.LookupStatus.Found, + value: [ + Cert.NodeType.Fork, + [Cert.NodeType.Labeled, label('1'), [Cert.NodeType.Leaf, label('42')]], + [ + Cert.NodeType.Pruned, + pruned('5ec92bd71f697eee773919200a9718c4719495a4c6bba52acc408bd79b4bf57f'), + ], + ], + }); + + const tree_d = (lookup_d as Cert.LookupResultFound).value as Cert.HashTree; + // a subtree at label `1` exists in the subtree at label `d` + expect(Cert.find_label(label('1'), tree_d)).toEqual({ + status: Cert.LookupStatus.Found, + value: [Cert.NodeType.Leaf, label('42')], + }); + expect(Cert.lookup_path([label('1')], tree_d)).toEqual({ + status: Cert.LookupStatus.Found, + value: label('42'), + }); + expect(Cert.lookup_path([label('d'), label('1')], tree)).toEqual({ + status: Cert.LookupStatus.Found, + value: label('42'), + }); + + // the rest of the subtree at label `d` is pruned, + // so any more lookups return Unknown + expect(Cert.find_label(label('2'), tree_d)).toEqual({ + status: Cert.LookupStatus.Unknown, + }); + expect(Cert.lookup_path([label('2')], tree_d)).toEqual({ + status: Cert.LookupStatus.Unknown, + }); + expect(Cert.lookup_path([label('d'), label('2')], tree)).toEqual({ + status: Cert.LookupStatus.Unknown, + }); + }); + + test('subtree_e', () => { + // a subtree at label `e` exists + expect(Cert.find_label(label('e'), tree)).toEqual({ + status: Cert.LookupStatus.Found, + value: [Cert.NodeType.Leaf, label('world')], + }); + expect(Cert.lookup_path([label('e')], tree)).toEqual({ + status: Cert.LookupStatus.Found, + value: label('world'), + }); + }); + + test('subtree_f', () => { + // there are no nodes between labels `e` and `g`, + // so the subtree at `f` is provably Absent + expect(Cert.find_label(label('f'), tree)).toEqual({ + status: Cert.LookupStatus.Absent, + }); + expect(Cert.lookup_path([label('f')], tree)).toEqual({ + status: Cert.LookupStatus.Absent, + }); + }); + + test('subtree_g', () => { + // a subtree at label `g` exists + const lookup_g = Cert.find_label(label('g'), tree); + expect(lookup_g).toEqual({ + status: Cert.LookupStatus.Found, + value: [Cert.NodeType.Empty], + }); + expect(Cert.lookup_path([label('g')], tree)).toEqual({ + status: Cert.LookupStatus.Found, + value: [Cert.NodeType.Empty], + }); + + // the subtree at label `g` is empty so any nested lookup are provably Absent + const tree_g = (lookup_g as Cert.LookupResultFound).value as Cert.HashTree; + expect(Cert.find_label(label('1'), tree_g)).toEqual({ + status: Cert.LookupStatus.Absent, + }); + expect(Cert.lookup_path([label('1')], tree_g)).toEqual({ + status: Cert.LookupStatus.Absent, + }); + expect(Cert.lookup_path([label('g'), label('1')], tree)).toEqual({ + status: Cert.LookupStatus.Absent, + }); + + expect(Cert.find_label(label('2'), tree_g)).toEqual({ + status: Cert.LookupStatus.Absent, + }); + expect(Cert.lookup_path([label('2')], tree_g)).toEqual({ + status: Cert.LookupStatus.Absent, + }); + expect(Cert.lookup_path([label('g'), label('2')], tree)).toEqual({ + status: Cert.LookupStatus.Absent, + }); + }); }); // The sample certificate for testing delegation is extracted from the response used in agent-rs tests, where they were taken @@ -147,7 +351,7 @@ const SAMPLE_CERT: string = const parseTimeFromCert = (cert: ArrayBuffer): Date => { const certObj = cbor.decode(new Uint8Array(cert)) as { tree: Cert.HashTree }; if (!certObj.tree) throw new Error('Invalid certificate'); - const lookup = lookupResultToBuffer(lookup_path(['time'], certObj.tree)); + const lookup = Cert.lookupResultToBuffer(Cert.lookup_path(['time'], certObj.tree)); if (!lookup) throw new Error('Invalid certificate'); return decodeTime(lookup); diff --git a/packages/agent/src/certificate.ts b/packages/agent/src/certificate.ts index b2fc0308..b2c77903 100644 --- a/packages/agent/src/certificate.ts +++ b/packages/agent/src/certificate.ts @@ -21,24 +21,22 @@ export interface Cert { delegation?: Delegation; } -const NodeId = { - Empty: 0, - Fork: 1, - Labeled: 2, - Leaf: 3, - Pruned: 4, -}; - -export type NodeIdType = typeof NodeId[keyof typeof NodeId]; +export enum NodeType { + Empty = 0, + Fork = 1, + Labeled = 2, + Leaf = 3, + Pruned = 4, +} -export { NodeId }; +export type NodeLabel = ArrayBuffer | Uint8Array; export type HashTree = - | [typeof NodeId.Empty] - | [typeof NodeId.Fork, HashTree, HashTree] - | [typeof NodeId.Labeled, ArrayBuffer, HashTree] - | [typeof NodeId.Leaf, ArrayBuffer] - | [typeof NodeId.Pruned, ArrayBuffer]; + | [NodeType.Empty] + | [NodeType.Fork, HashTree, HashTree] + | [NodeType.Labeled, NodeLabel, HashTree] + | [NodeType.Leaf, NodeLabel] + | [NodeType.Pruned, NodeLabel]; /** * Make a human readable string out of a hash tree. @@ -60,9 +58,9 @@ export function hashTreeToString(tree: HashTree): string { } switch (tree[0]) { - case NodeId.Empty: + case NodeType.Empty: return '()'; - case NodeId.Fork: { + case NodeType.Fork: { if (tree[1] instanceof Array && tree[2] instanceof ArrayBuffer) { const left = hashTreeToString(tree[1]); const right = hashTreeToString(tree[2]); @@ -71,7 +69,7 @@ export function hashTreeToString(tree: HashTree): string { throw new Error('Invalid tree structure for fork'); } } - case NodeId.Labeled: { + case NodeType.Labeled: { if (tree[1] instanceof ArrayBuffer && tree[2] instanceof ArrayBuffer) { const label = labelToString(tree[1]); const sub = hashTreeToString(tree[2]); @@ -80,7 +78,7 @@ export function hashTreeToString(tree: HashTree): string { throw new Error('Invalid tree structure for labeled'); } } - case NodeId.Leaf: { + case NodeType.Leaf: { if (!tree[1]) { throw new Error('Invalid tree structure for leaf'); } else if (Array.isArray(tree[1])) { @@ -88,7 +86,7 @@ export function hashTreeToString(tree: HashTree): string { } return `leaf(...${tree[1].byteLength} bytes)`; } - case NodeId.Pruned: { + case NodeType.Pruned: { if (!tree[1]) { throw new Error('Invalid tree structure for pruned'); } else if (Array.isArray(tree[1])) { @@ -108,7 +106,7 @@ interface Delegation extends Record { certificate: ArrayBuffer; } -function isBufferEqual(a: ArrayBuffer, b: ArrayBuffer): boolean { +export function isBufferEqual(a: ArrayBuffer, b: ArrayBuffer): boolean { if (a.byteLength !== b.byteLength) { return false; } @@ -122,6 +120,17 @@ function isBufferEqual(a: ArrayBuffer, b: ArrayBuffer): boolean { return true; } +function isBufferGreaterThan(a: ArrayBuffer, b: ArrayBuffer): boolean { + const a8 = new Uint8Array(a); + const b8 = new Uint8Array(b); + for (let i = 0; i < a8.length; i++) { + if (a8[i] > b8[i]) { + return true; + } + } + return false; +} + type VerifyFunc = (pk: Uint8Array, sig: Uint8Array, msg: Uint8Array) => Promise | boolean; export interface CreateCertificateOptions { @@ -321,14 +330,19 @@ function extractDER(buf: ArrayBuffer): ArrayBuffer { * @param {ArrayBuffer | HashTree | undefined} result - the result of a lookup * @returns ArrayBuffer or Undefined */ -export function lookupResultToBuffer( - result: ArrayBuffer | HashTree | undefined, -): ArrayBuffer | undefined { - if (result instanceof ArrayBuffer) { - return result; - } else if (result instanceof Uint8Array) { - return result.buffer; +export function lookupResultToBuffer(result: LookupResult): ArrayBuffer | undefined { + if (result.status !== LookupStatus.Found) { + return undefined; + } + + if (result.value instanceof ArrayBuffer) { + return result.value; + } + + if (result.value instanceof Uint8Array) { + return result.value.buffer; } + return undefined; } @@ -337,13 +351,13 @@ export function lookupResultToBuffer( */ export async function reconstruct(t: HashTree): Promise { switch (t[0]) { - case NodeId.Empty: + case NodeType.Empty: return hash(domain_sep('ic-hashtree-empty')); - case NodeId.Pruned: + case NodeType.Pruned: return t[1] as ArrayBuffer; - case NodeId.Leaf: + case NodeType.Leaf: return hash(concat(domain_sep('ic-hashtree-leaf'), t[1] as ArrayBuffer)); - case NodeId.Labeled: + case NodeType.Labeled: return hash( concat( domain_sep('ic-hashtree-labeled'), @@ -351,7 +365,7 @@ export async function reconstruct(t: HashTree): Promise { await reconstruct(t[2] as HashTree), ), ); - case NodeId.Fork: + case NodeType.Fork: return hash( concat( domain_sep('ic-hashtree-fork'), @@ -370,38 +384,97 @@ function domain_sep(s: string): ArrayBuffer { return concat(len, str); } -/** - * @param path - * @param tree - */ -export function lookup_path( - path: Array, - tree: HashTree, -): ArrayBuffer | HashTree | undefined { +export enum LookupStatus { + Unknown = 'unknown', + Absent = 'absent', + Found = 'found', +} + +export interface LookupResultAbsent { + status: LookupStatus.Absent; +} + +export interface LookupResultUnknown { + status: LookupStatus.Unknown; +} + +export interface LookupResultFound { + status: LookupStatus.Found; + value: ArrayBuffer | HashTree; +} + +export type LookupResult = LookupResultAbsent | LookupResultUnknown | LookupResultFound; + +enum LabelLookupStatus { + Less = 'less', + Greater = 'greater', +} + +interface LookupResultGreater { + status: LabelLookupStatus.Greater; +} + +interface LookupResultLess { + status: LabelLookupStatus.Less; +} + +type LabelLookupResult = LookupResult | LookupResultGreater | LookupResultLess; + +export function lookup_path(path: Array, tree: HashTree): LookupResult { if (path.length === 0) { switch (tree[0]) { - case NodeId.Leaf: { - // should not be undefined - if (!tree[1]) throw new Error('Invalid tree structure for leaf'); + case NodeType.Leaf: { + if (!tree[1]) { + throw new Error('Invalid tree structure for leaf'); + } + if (tree[1] instanceof ArrayBuffer) { - return tree[1]; - } else if (tree[1] instanceof Uint8Array) { - return tree[1].buffer; - } else return tree[1]; - } - case NodeId.Fork: { - return tree; + return { + status: LookupStatus.Found, + value: tree[1], + }; + } + + if (tree[1] instanceof Uint8Array) { + return { + status: LookupStatus.Found, + value: tree[1].buffer, + }; + } + + return { + status: LookupStatus.Found, + value: tree[1], + }; } + default: { - return tree; + return { + status: LookupStatus.Found, + value: tree, + }; } } } const label = typeof path[0] === 'string' ? new TextEncoder().encode(path[0]) : path[0]; - const t = find_label(label, flatten_forks(tree)); - if (t) { - return lookup_path(path.slice(1), t); + const lookupResult = find_label(label, tree); + + switch (lookupResult.status) { + case LookupStatus.Found: { + return lookup_path(path.slice(1), lookupResult.value as HashTree); + } + + case LabelLookupStatus.Greater: + case LabelLookupStatus.Less: { + return { + status: LookupStatus.Absent, + }; + } + + default: { + return lookupResult; + } } } @@ -412,26 +485,108 @@ export function lookup_path( */ export function flatten_forks(t: HashTree): HashTree[] { switch (t[0]) { - case NodeId.Empty: + case NodeType.Empty: return []; - case NodeId.Fork: + case NodeType.Fork: return flatten_forks(t[1] as HashTree).concat(flatten_forks(t[2] as HashTree)); default: return [t]; } } -function find_label(l: ArrayBuffer, trees: HashTree[]): HashTree | undefined { - if (trees.length === 0) { - return undefined; - } - for (const t of trees) { - if (t[0] === NodeId.Labeled) { - const p = t[1] as ArrayBuffer; - if (isBufferEqual(l, p)) { - return t[2]; +export function find_label(label: ArrayBuffer, tree: HashTree): LabelLookupResult { + switch (tree[0]) { + // if we have a labelled node, compare the node's label to the one we are + // looking for + case NodeType.Labeled: + // if the label we're searching for is greater than this node's label, + // we need to keep searching + if (isBufferGreaterThan(label, tree[1])) { + return { + status: LabelLookupStatus.Greater, + }; } - } + + // if the label we're searching for is equal this node's label, we can + // stop searching and return the found node + if (isBufferEqual(label, tree[1])) { + return { + status: LookupStatus.Found, + value: tree[2], + }; + } + + // if the label we're searching for is not greater than or equal to this + // node's label, then it's less than this node's label, and we can stop + // searching because we've looked too far + return { + status: LabelLookupStatus.Less, + }; + + // if we have a fork node, we need to search both sides, starting with the left + case NodeType.Fork: + // search in the left node + const leftLookupResult = find_label(label, tree[1]); + + switch (leftLookupResult.status) { + // if the label we're searching for is greater than the left node lookup, + // we need to search the right node + case LabelLookupStatus.Greater: { + const rightLookupResult = find_label(label, tree[2]); + + // if the label we're searching for is less than the right node lookup, + // then we can stop searching and say that the label is provably Absent + if (rightLookupResult.status === LabelLookupStatus.Less) { + return { + status: LookupStatus.Absent, + }; + } + + // if the label we're searching for is less than or equal to the right + // node lookup, then we let the caller handle it + return rightLookupResult; + } + + // if the left node returns an uncertain result, we need to search the + // right node + case LookupStatus.Unknown: { + let rightLookupResult = find_label(label, tree[2]); + + // if the label we're searching for is less than the right node lookup, + // then we also need to return an uncertain result + if (rightLookupResult.status === LabelLookupStatus.Less) { + return { + status: LookupStatus.Unknown, + }; + } + + // if the label we're searching for is less than or equal to the right + // node lookup, then we let the caller handle it + return rightLookupResult; + } + + // if the label we're searching for is not greater than the left node + // lookup, or the result is not uncertain, we stop searching and return + // whatever the result of the left node lookup was, which can be either + // Found or Absent + default: { + return leftLookupResult; + } + } + + // if we encounter a Pruned node, we can't know for certain if the label + // we're searching for is present or not + case NodeType.Pruned: + return { + status: LookupStatus.Unknown, + }; + + // if the current node is Empty, or a Leaf, we can stop searching because + // we know for sure that the label we're searching for is not present + default: + return { + status: LookupStatus.Absent, + }; } } @@ -449,11 +604,11 @@ export function check_canister_ranges(params: { const { canisterId, subnetId, tree } = params; const rangeLookup = lookup_path(['subnet', subnetId.toUint8Array(), 'canister_ranges'], tree); - if (!rangeLookup || !(rangeLookup instanceof ArrayBuffer)) { + if (rangeLookup.status !== LookupStatus.Found || !(rangeLookup.value instanceof ArrayBuffer)) { throw new Error(`Could not find canister ranges for subnet ${subnetId}`); } - const ranges_arr: Array<[Uint8Array, Uint8Array]> = cbor.decode(rangeLookup); + const ranges_arr: Array<[Uint8Array, Uint8Array]> = cbor.decode(rangeLookup.value); const ranges: Array<[Principal, Principal]> = ranges_arr.map(v => [ Principal.fromUint8Array(v[0]), Principal.fromUint8Array(v[1]), From 2ef1692163e8fe947a8a38cc4b5421edffa285fe Mon Sep 17 00:00:00 2001 From: NathanosDev Date: Tue, 7 May 2024 19:52:09 +0200 Subject: [PATCH 2/2] feat: propogate lookup result to public interface --- docs/CHANGELOG.md | 4 +++ e2e/node/basic/basic.test.ts | 38 +++++++++++++++++----- packages/agent/src/canisterStatus/index.ts | 3 +- packages/agent/src/certificate.test.ts | 20 ++---------- packages/agent/src/certificate.ts | 32 ++++++------------ packages/agent/src/polling/index.ts | 14 +++++--- packages/assets/src/index.ts | 12 +++++-- 7 files changed, 65 insertions(+), 58 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 75facd80..9a2d8b70 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- feat!: add support for proof of absence in Certificate lookups + ## [1.3.0] - 2024-05-01 ### Added diff --git a/e2e/node/basic/basic.test.ts b/e2e/node/basic/basic.test.ts index 4779f4ea..4385a1a8 100644 --- a/e2e/node/basic/basic.test.ts +++ b/e2e/node/basic/basic.test.ts @@ -1,4 +1,10 @@ -import { ActorMethod, Certificate, getManagementCanister } from '@dfinity/agent'; +import { + ActorMethod, + Certificate, + LookupResultFound, + LookupStatus, + getManagementCanister, +} from '@dfinity/agent'; import { IDL } from '@dfinity/candid'; import { Principal } from '@dfinity/principal'; import agent from '../utils/agent'; @@ -18,14 +24,21 @@ test('read_state', async () => { rootKey: resolvedAgent.rootKey, canisterId: canisterId, }); - expect(cert.lookup([new TextEncoder().encode('Time')])).toBe(undefined); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const rawTime = cert.lookup(path)!; + expect(cert.lookup([new TextEncoder().encode('Time')])).toEqual({ status: LookupStatus.Unknown }); + + let rawTime = cert.lookup(path); + + expect(rawTime.status).toEqual(LookupStatus.Found); + rawTime = rawTime as LookupResultFound; + + expect(rawTime.value).toBeInstanceOf(ArrayBuffer); + rawTime.value = rawTime.value as ArrayBuffer; + const decoded = IDL.decode( [IDL.Nat], new Uint8Array([ ...new TextEncoder().encode('DIDL\x00\x01\x7d'), - ...(new Uint8Array(rawTime) || []), + ...(new Uint8Array(rawTime.value) || []), ]), )[0]; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -54,14 +67,21 @@ test('read_state with passed request', async () => { rootKey: resolvedAgent.rootKey, canisterId: canisterId, }); - expect(cert.lookup([new TextEncoder().encode('Time')])).toBe(undefined); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const rawTime = cert.lookup(path)!; + expect(cert.lookup([new TextEncoder().encode('Time')])).toEqual({ status: LookupStatus.Unknown }); + + let rawTime = cert.lookup(path); + + expect(rawTime.status).toEqual(LookupStatus.Found); + rawTime = rawTime as LookupResultFound; + + expect(rawTime.value).toBeInstanceOf(ArrayBuffer); + rawTime.value = rawTime.value as ArrayBuffer; + const decoded = IDL.decode( [IDL.Nat], new Uint8Array([ ...new TextEncoder().encode('DIDL\x00\x01\x7d'), - ...(new Uint8Array(rawTime) || []), + ...(new Uint8Array(rawTime.value) || []), ]), )[0]; // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/packages/agent/src/canisterStatus/index.ts b/packages/agent/src/canisterStatus/index.ts index 2bed3789..c46ba2ca 100644 --- a/packages/agent/src/canisterStatus/index.ts +++ b/packages/agent/src/canisterStatus/index.ts @@ -11,6 +11,7 @@ import { check_canister_ranges, LookupStatus, lookup_path, + lookupResultToBuffer, } from '../certificate'; import { toHex } from '../utils/buffer'; import * as Cbor from '../cbor'; @@ -160,7 +161,7 @@ export const request = async (options: { } else { return { path: path, - data: cert.lookup(encodePath(path, canisterId)), + data: lookupResultToBuffer(cert.lookup(encodePath(path, canisterId))), }; } }; diff --git a/packages/agent/src/certificate.test.ts b/packages/agent/src/certificate.test.ts index 7e925f37..74c3bf23 100644 --- a/packages/agent/src/certificate.test.ts +++ b/packages/agent/src/certificate.test.ts @@ -5,7 +5,7 @@ */ import * as cbor from './cbor'; import * as Cert from './certificate'; -import { fromHex, toHex } from './utils/buffer'; +import { bufEquals, fromHex, toHex } from './utils/buffer'; import { Principal } from '@dfinity/principal'; import { decodeTime } from './utils/leb'; import { readFileSync } from 'fs'; @@ -19,23 +19,7 @@ function pruned(str: string): ArrayBuffer { return fromHex(str); } -function bufferEqualityTester(a: unknown, b: unknown): boolean | undefined { - if (a instanceof ArrayBuffer && b instanceof ArrayBuffer) { - return Cert.isBufferEqual(a, b); - } - - if (a instanceof ArrayBuffer && b instanceof Uint8Array) { - return Cert.isBufferEqual(a, b.buffer); - } - - if (a instanceof Uint8Array && b instanceof ArrayBuffer) { - return Cert.isBufferEqual(a.buffer, b); - } - - return undefined; -} - -(expect as any).addEqualityTesters([bufferEqualityTester]); +(expect as any).addEqualityTesters([bufEquals]); // Root public key for the IC main net, encoded as hex const IC_ROOT_KEY = diff --git a/packages/agent/src/certificate.ts b/packages/agent/src/certificate.ts index b2c77903..a8cdf551 100644 --- a/packages/agent/src/certificate.ts +++ b/packages/agent/src/certificate.ts @@ -1,7 +1,7 @@ import * as cbor from './cbor'; import { AgentError } from './errors'; import { hash } from './request_id'; -import { concat, fromHex, toHex } from './utils/buffer'; +import { bufEquals, concat, fromHex, toHex } from './utils/buffer'; import { Principal } from '@dfinity/principal'; import * as bls from './utils/bls'; import { decodeTime } from './utils/leb'; @@ -106,20 +106,6 @@ interface Delegation extends Record { certificate: ArrayBuffer; } -export function isBufferEqual(a: ArrayBuffer, b: ArrayBuffer): boolean { - if (a.byteLength !== b.byteLength) { - return false; - } - const a8 = new Uint8Array(a); - const b8 = new Uint8Array(b); - for (let i = 0; i < a8.length; i++) { - if (a8[i] !== b8[i]) { - return false; - } - } - return true; -} - function isBufferGreaterThan(a: ArrayBuffer, b: ArrayBuffer): boolean { const a8 = new Uint8Array(a); const b8 = new Uint8Array(b); @@ -208,12 +194,12 @@ export class Certificate { this.cert = cbor.decode(new Uint8Array(certificate)); } - public lookup(path: Array): ArrayBuffer | undefined { + public lookup(path: Array): LookupResult { // constrain the type of the result, so that empty HashTree is undefined - return lookupResultToBuffer(lookup_path(path, this.cert.tree)); + return lookup_path(path, this.cert.tree); } - public lookup_label(label: ArrayBuffer): ArrayBuffer | HashTree | undefined { + public lookup_label(label: ArrayBuffer): LookupResult { return this.lookup([label]); } @@ -225,7 +211,7 @@ export class Certificate { const msg = concat(domain_sep('ic-state-root'), rootHash); let sigVer = false; - const lookupTime = this.lookup(['time']); + const lookupTime = lookupResultToBuffer(this.lookup(['time'])); if (!lookupTime) { // Should never happen - time is always present in IC certificates throw new CertificateVerificationError('Certificate does not contain a time'); @@ -297,7 +283,9 @@ export class Certificate { )}`, ); } - const publicKeyLookup = cert.lookup(['subnet', d.subnet_id, 'public_key']); + const publicKeyLookup = lookupResultToBuffer( + cert.lookup(['subnet', d.subnet_id, 'public_key']), + ); if (!publicKeyLookup) { throw new Error(`Could not find subnet key for subnet 0x${toHex(d.subnet_id)}`); } @@ -316,7 +304,7 @@ function extractDER(buf: ArrayBuffer): ArrayBuffer { throw new TypeError(`BLS DER-encoded public key must be ${expectedLength} bytes long`); } const prefix = buf.slice(0, DER_PREFIX.byteLength); - if (!isBufferEqual(prefix, DER_PREFIX)) { + if (!bufEquals(prefix, DER_PREFIX)) { throw new TypeError( `BLS DER-encoded public key is invalid. Expect the following prefix: ${DER_PREFIX}, but get ${prefix}`, ); @@ -509,7 +497,7 @@ export function find_label(label: ArrayBuffer, tree: HashTree): LabelLookupResul // if the label we're searching for is equal this node's label, we can // stop searching and return the found node - if (isBufferEqual(label, tree[1])) { + if (bufEquals(label, tree[1])) { return { status: LookupStatus.Found, value: tree[2], diff --git a/packages/agent/src/polling/index.ts b/packages/agent/src/polling/index.ts index e964f61a..27dba4bb 100644 --- a/packages/agent/src/polling/index.ts +++ b/packages/agent/src/polling/index.ts @@ -1,6 +1,6 @@ import { Principal } from '@dfinity/principal'; import { Agent, RequestStatusResponseStatus } from '../agent'; -import { Certificate, CreateCertificateOptions } from '../certificate'; +import { Certificate, CreateCertificateOptions, lookupResultToBuffer } from '../certificate'; import { RequestId } from '../request_id'; import { toHex } from '../utils/buffer'; @@ -42,7 +42,7 @@ export async function pollForResponse( canisterId: canisterId, blsVerify, }); - const maybeBuf = cert.lookup([...path, new TextEncoder().encode('status')]); + const maybeBuf = lookupResultToBuffer(cert.lookup([...path, new TextEncoder().encode('status')])); let status; if (typeof maybeBuf === 'undefined') { // Missing requestId means we need to wait @@ -53,7 +53,7 @@ export async function pollForResponse( switch (status) { case RequestStatusResponseStatus.Replied: { - return cert.lookup([...path, 'reply'])!; + return lookupResultToBuffer(cert.lookup([...path, 'reply']))!; } case RequestStatusResponseStatus.Received: @@ -64,8 +64,12 @@ export async function pollForResponse( return pollForResponse(agent, canisterId, requestId, strategy, currentRequest); case RequestStatusResponseStatus.Rejected: { - const rejectCode = new Uint8Array(cert.lookup([...path, 'reject_code'])!)[0]; - const rejectMessage = new TextDecoder().decode(cert.lookup([...path, 'reject_message'])!); + const rejectCode = new Uint8Array( + lookupResultToBuffer(cert.lookup([...path, 'reject_code']))!, + )[0]; + const rejectMessage = new TextDecoder().decode( + lookupResultToBuffer(cert.lookup([...path, 'reject_message']))!, + ); throw new Error( `Call was rejected:\n` + ` Request ID: ${toHex(requestId)}\n` + diff --git a/packages/assets/src/index.ts b/packages/assets/src/index.ts index 9c4e9161..5c20310b 100644 --- a/packages/assets/src/index.ts +++ b/packages/assets/src/index.ts @@ -9,6 +9,7 @@ import { HashTree, lookup_path, lookupResultToBuffer, + LookupStatus, reconstruct, uint8ToBuf, } from '@dfinity/agent'; @@ -540,7 +541,12 @@ class Asset { } // Check certificate time - const decodedTime = lebDecode(new PipeArrayBuffer(cert.lookup(['time']))); + const timeLookup = cert.lookup(['time']); + if (timeLookup.status !== LookupStatus.Found || !(timeLookup.value instanceof ArrayBuffer)) { + return false; + } + + const decodedTime = lebDecode(new PipeArrayBuffer(timeLookup.value)); const certTime = Number(decodedTime / BigInt(1_000_000)); // Convert from nanos to millis const now = Date.now(); const maxCertTimeOffset = 300_000; // 5 min @@ -552,13 +558,13 @@ class Asset { const reconstructed = await reconstruct(hashTree); const witness = cert.lookup(['canister', canisterId.toUint8Array(), 'certified_data']); - if (!witness) { + if (witness.status !== LookupStatus.Found || !(witness.value instanceof ArrayBuffer)) { // Could not find certified data for this canister in the certificate return false; } // First validate that the Tree is as good as the certification - if (compare(witness, reconstructed) !== 0) { + if (compare(witness.value, reconstructed) !== 0) { // Witness != Tree passed in ic-certification return false; }