From 6a822dc1ace3603a7b893ad55b3162286e10a3bd Mon Sep 17 00:00:00 2001 From: Meno Abels Date: Wed, 11 Sep 2024 17:57:14 +0200 Subject: [PATCH] chore: template for internal-layer-protocol --- src/blockstore/fp-envelope.ts | 116 +++++++++++++++++++++++++++ src/blockstore/index.ts | 2 + src/blockstore/store.ts | 23 +++++- src/runtime/gateways/file/gateway.ts | 14 +++- tests/blockstore/fp-envelope.test.ts | 56 +++++++++++++ 5 files changed, 208 insertions(+), 3 deletions(-) create mode 100644 src/blockstore/fp-envelope.ts create mode 100644 tests/blockstore/fp-envelope.test.ts diff --git a/src/blockstore/fp-envelope.ts b/src/blockstore/fp-envelope.ts new file mode 100644 index 00000000..2efb3882 --- /dev/null +++ b/src/blockstore/fp-envelope.ts @@ -0,0 +1,116 @@ +import { CID } from "multiformats"; +import { encode, decode } from "cborg"; +import { Result } from "@adviser/cement" + +export interface FPEnvelope { + readonly type: string; // "car" | "file" | "meta" | "wal" + readonly payload: T +} + +export interface FPEnvelopeCar extends FPEnvelope { + readonly type: "car"; +} + +export interface FPEnvelopeFile extends FPEnvelope { + readonly type: "file"; +} + +export interface FPMeta { + readonly cid: string; + readonly data: Uint8Array; + readonly parents: string[]; +} + +export interface FPEnvelopeMeta extends FPEnvelope { + readonly type: "meta"; +} + +export interface FPWALCarsOps { + readonly cars: CID[]; +} +export interface FPWAL { + // fileOperations: any[]; will be added with connector-fixes + // noLoaderOps: any[]; will be added with connector-fixes + readonly operations: FPWALCarsOps[]; +} +export interface FPEnvelopeWAL extends FPEnvelope { + readonly type: "wal"; +} + +export function WAL2FPMsg(fpwal: FPWAL): Uint8Array { + return encode({ type: "wal", payload: JSON.parse(JSON.stringify(fpwal)) } as FPEnvelopeWAL); +} + +export function FPMsg2WAL(fpmsg: Uint8Array): Result { + const renv = FPMsgMatch2Envelope(fpmsg, "wal"); + if (renv.isErr()) { + return Result.Err(renv.Err()); + } + const convertCids = renv.Ok().payload as FPWAL; + for (const op of convertCids.operations) { + const cars = [] + for (const strCid of op.cars) { + for (const cidVal of Object.values(strCid)) { + cars.push(CID.parse(cidVal)); + } + } + (op as {cars: CID[]}).cars = cars; + } + return Result.Ok(renv.Ok().payload as FPWAL); +} + +export function Meta2FPMsg(fpmeta: FPMeta): Uint8Array { + return encode({ type: "meta", payload: fpmeta } as FPEnvelopeMeta); +} + +export function FPMsg2Meta(fpmsg: Uint8Array): Result { + const renv = FPMsgMatch2Envelope(fpmsg, "meta"); + if (renv.isErr()) { + return Result.Err(renv.Err()); + } + return Result.Ok(renv.Ok().payload as FPMeta); +} + +export function Car2FPMsg(fpcar: Uint8Array): Uint8Array { + return encode({ type: "car", payload: fpcar } as FPEnvelopeCar); +} + +export function FPMsg2Car(fpmsg: Uint8Array): Result { + const renv = FPMsgMatch2Envelope(fpmsg, "car"); + if (renv.isErr()) { + return Result.Err(renv.Err()); + } + return Result.Ok(renv.Ok().payload as Uint8Array); +} + +export function File2FPMsg(fpfile: Uint8Array): Uint8Array { + return encode({ type: "file", payload: fpfile } as FPEnvelopeFile); +} + +export function FPMsg2File(fpmsg: Uint8Array): Result { + const renv = FPMsgMatch2Envelope(fpmsg, "file"); + if (renv.isErr()) { + return Result.Err(renv.Err()); + } + return Result.Ok(renv.Ok().payload as Uint8Array); +} + +export function FPMsgMatch2Envelope(fpmsg: Uint8Array, ...types: string[]): Result> { + let env: FPEnvelope; + try { + env = decode(fpmsg); + } catch (e) { + return Result.Err(`failed to decode envelope: ${e}`); + } + if (typeof env !== "object") { + return Result.Err(`expected envelope to be an object`); + } + if (typeof env.type !== "string") { + return Result.Err(`expected type to be a string`); + } + if (types.length > 0 && !types.includes(env.type)) { + return Result.Err(`expected type to be ${types}`); + } + // need to check if the payload is a valid WAL + return Result.Ok(env); +} \ No newline at end of file diff --git a/src/blockstore/index.ts b/src/blockstore/index.ts index 41179748..02c56621 100644 --- a/src/blockstore/index.ts +++ b/src/blockstore/index.ts @@ -4,6 +4,8 @@ export * from "./store-factory.js"; export * from "./gateway.js"; export * from "./fragment-gateway.js"; +export * from "./fp-envelope.js" + import { type Connectable } from "./connection-base.js"; export { Connectable }; diff --git a/src/blockstore/store.ts b/src/blockstore/store.ts index 99e30eaf..fb1b3535 100644 --- a/src/blockstore/store.ts +++ b/src/blockstore/store.ts @@ -30,6 +30,7 @@ import { keyedCryptoFactory } from "../runtime/keyed-crypto.js"; import { KeyBag } from "../runtime/key-bag.js"; import { FragmentGateway } from "./fragment-gateway.js"; import { Link } from "multiformats"; +import { FPEnvelopeCar, FPEnvelopeFile, FPMsg2Car, FPMsgMatch2Envelope } from "./fp-envelope.js"; function guardVersion(url: URI): Result { if (!url.hasParam("version")) { @@ -287,7 +288,19 @@ export class DataStoreImpl extends BaseStoreImpl implements DataStore { if (res.isErr()) { throw res.Err(); } - return { cid, bytes: res.Ok() }; + const rfpenv = FPMsgMatch2Envelope(res.Ok(), "car", "file"); + if (rfpenv.isErr()) { + throw this.logger.Error().Err(rfpenv.Err()).Msg("got error from FPMsgMatch2Envelope").AsError(); + } + const fpenv = rfpenv.Ok() as FPEnvelopeFile|FPEnvelopeCar + switch (fpenv.type) { + case "car": + return { cid, bytes: fpenv.payload }; + case "file": + return { cid, bytes: fpenv.payload }; + default: + throw this.logger.Error().Msg("unexpected type").AsError(); + } } // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -297,7 +310,13 @@ export class DataStoreImpl extends BaseStoreImpl implements DataStore { if (url.isErr()) { throw this.logger.Error().Err(url.Err()).Ref("cid", car.cid).Msg("got error from gateway.buildUrl").AsError(); } - const res = await this.gateway.put(url.Ok(), car.bytes); + // without URL changes in super-this branch we + // can distinguish between car and file + const rfpenv = FPMsg2Car(car.bytes); + if (rfpenv.isErr()) { + throw this.logger.Error().Err(rfpenv.Err()).Msg("got error from FPMsg2Car").AsError(); + } + const res = await this.gateway.put(url.Ok(), rfpenv.Ok()); if (res.isErr()) { throw this.logger.Error().Err(res.Err()).Msg("got error from gateway.put").AsError(); } diff --git a/src/runtime/gateways/file/gateway.ts b/src/runtime/gateways/file/gateway.ts index 972489a7..d0b7d339 100644 --- a/src/runtime/gateways/file/gateway.ts +++ b/src/runtime/gateways/file/gateway.ts @@ -4,6 +4,7 @@ import { ensureLogger, exceptionWrapper, isNotFoundError, NotFoundError } from " import { Gateway, GetResult, TestGateway } from "../../../blockstore/gateway.js"; import { getFileName, getFileSystem, getPath } from "./utils.js"; import { SuperThis, SysFileSystem } from "../../../types.js"; +import { FPEnvelopeCar, FPEnvelopeFile, FPMsg2Car, FPMsgMatch2Envelope } from "../../../blockstore/fp-envelope.js"; const versionFiles = new KeyedResolvOnce(); @@ -77,10 +78,18 @@ export class FileGateway implements Gateway { } async put(url: URI, body: Uint8Array): Promise> { + const rbuf = FPMsgMatch2Envelope(body) as Result; + if (rbuf.isErr()) { + return Result.Err(rbuf.Err()); + } + let payload = body + if (["car", "file"].includes(rbuf.Ok().type)) { + payload = rbuf.Ok().payload + } return exception2Result(async () => { const file = await this.getFilePath(url); this.logger.Debug().Str("url", url.toString()).Str("file", file).Msg("put"); - await this.fs.writefile(file, body); + await this.fs.writefile(file, payload); }); } @@ -90,6 +99,9 @@ export class FileGateway implements Gateway { try { const res = await this.fs.readfile(file); this.logger.Debug().Url(url.asURL()).Str("file", file).Msg("get"); + if (url.getParam("store") === "data") { + return FPMsg2Car(res); + } return Result.Ok(new Uint8Array(res)); } catch (e: unknown) { // this.logger.Error().Err(e).Str("file", file).Msg("get"); diff --git a/tests/blockstore/fp-envelope.test.ts b/tests/blockstore/fp-envelope.test.ts new file mode 100644 index 00000000..ef4fc1d1 --- /dev/null +++ b/tests/blockstore/fp-envelope.test.ts @@ -0,0 +1,56 @@ +import { encode } from "cborg"; +import { bs, Result } from '@fireproof/core' +import { CID } from "multiformats"; + +it("unknown bytes", () => { + expect(bs.FPMsgMatch2Envelope(Uint8Array.from([1, 2, 3]), "bla").Err().message).toStrictEqual( + "failed to decode envelope: Error: CBOR decode error: too many terminals, data makes no sense" + ); +}); + +it("unknown type", () => { + expect(bs.FPMsgMatch2Envelope(encode({ type: "blax", payload: 4 }), "bla")) + .toStrictEqual(Result.Err("expected type to be bla")); +}) + +it("no type", () => { + expect(bs.FPMsgMatch2Envelope(encode({ type: "blax", payload: 4 }))) + .toStrictEqual(Result.Ok({ type: "blax", payload: 4 })); +}) + +it("car type", () => { + expect(bs.FPMsg2Car(bs.Car2FPMsg(Uint8Array.from([1, 2, 3]))).Ok()).toStrictEqual(Uint8Array.from([1, 2, 3])); +}) + +it("file type", () => { + expect(bs.FPMsg2File(bs.File2FPMsg(Uint8Array.from([1, 2, 3]))).Ok()).toStrictEqual(Uint8Array.from([1, 2, 3])); +}) + +it("meta type", () => { + const ref = { + cid: "CID", + data: Uint8Array.from([1, 2, 3]), + parents: ["C1", "C2"] + }; + expect(bs.FPMsg2Meta(bs.Meta2FPMsg(ref)).Ok()).toStrictEqual(ref); +}) + +it("wal type", () => { + const ref: bs.FPWAL = { + operations: [ + { + cars: [ + CID.parse("bag4yvqabciqdzvfxrxfi6feubspyz666zegmp3z5w556mr4ykya2kkdm22r7pyy") + ] + }, + { + cars: [ + CID.parse("bag4yvqabciqd2ul2tw4mdcpvfq2pdqhvnqp2ktuyrtcl3j3gwhxbjzjt62xzeaq") + ] + } + ] + }; + const res = bs.FPMsg2WAL(bs.WAL2FPMsg(ref)).Ok(); + expect(res).toStrictEqual(ref); + expect(res.operations[0].cars[0].version).toStrictEqual(1); +}) \ No newline at end of file