diff --git a/package-fireproof-core.json b/package-fireproof-core.json index 46e663ab9..3dbad73bd 100644 --- a/package-fireproof-core.json +++ b/package-fireproof-core.json @@ -53,17 +53,12 @@ }, "dependencies": { "@adviser/cement": "from-package-json", - "multiformats": "from-package-json", + "@fireproof/vendor": "from-package-json", "@ipld/unixfs": "from-package-json", - "@ipld/car": "from-package-json", - "@ipld/dag-json": "from-package-json", - "@ipld/dag-cbor": "from-package-json", - "@web3-storage/pail": "from-package-json", - "cborg": "from-package-json", + "multiformats": "from-package-json", "charwise": "from-package-json", "prolly-trees": "from-package-json", "idb": "from-package-json", - "ipfs-unixfs-exporter": "from-package-json", "memfs": "from-package-json", "p-limit": "from-package-json", "p-map": "from-package-json", diff --git a/package-use-fireproof.json b/package-use-fireproof.json index ab6d89dc8..3a1a186e1 100644 --- a/package-use-fireproof.json +++ b/package-use-fireproof.json @@ -29,20 +29,7 @@ "gptdoc": "Fireproof/React/Usage: import { useLiveQuery, useDocument } from 'use-fireproof'; function App() { const result = useLiveQuery(doc => doc.word, { limit: 10 }); const [{ count }, setDoc, saveDoc] = useDocument({_id: 'count', count: 0}); return (<>

{count} changes

saveDoc({count: count + 1})} onSubmit={e => useLiveQuery.database.put({word: e.target.value})} />)}", "dependencies": { "react": "from-package-json", - "multiformats": "from-package-json", - "@fireproof/core": "from-package-json", - "@adviser/cement": "from-package-json", - "charwise": "from-package-json", - "prolly-trees": "from-package-json", - "ipfs-unixfs-exporter": "from-package-json", - "idb": "from-package-json", - "p-limit": "from-package-json", - "p-map": "from-package-json", - "p-retry": "from-package-json", - "cborg": "from-package-json", - "@web3-storage/pail": "from-package-json", - "@ipld/unixfs": "from-package-json", - "@ipld/car": "from-package-json" + "@fireproof/core": "from-package-json" }, "peerDependencies": { "react": "from-package-json", diff --git a/package.json b/package.json index aee9449b1..030afab3c 100644 --- a/package.json +++ b/package.json @@ -78,17 +78,12 @@ "homepage": "https://github.com/fireproof-storage/fireproof#readme", "dependencies": { "@adviser/cement": "^0.2.40", - "@ipld/car": "^5.3.3", - "@ipld/dag-cbor": "^9.2.2", - "@ipld/dag-json": "^10.2.3", + "@fireproof/vendor": "^1.0.0", "@ipld/unixfs": "^3.0.0", - "@web3-storage/pail": "^0.6.0", - "cborg": "^4.2.6", "charwise": "^3.0.1", "cross-fetch": "^4.0.0", "esbuild-plugin-replace": "^1.4.0", "idb": "^8.0.0", - "ipfs-unixfs-exporter": "^13.6.1", "memfs": "^4.14.1", "multiformats": "^13.3.1", "p-limit": "^6.1.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3c485d188..4543a6c62 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,24 +11,12 @@ importers: '@adviser/cement': specifier: ^0.2.40 version: 0.2.40(typescript@5.7.2) - '@ipld/car': - specifier: ^5.3.3 - version: 5.3.3 - '@ipld/dag-cbor': - specifier: ^9.2.2 - version: 9.2.2 - '@ipld/dag-json': - specifier: ^10.2.3 - version: 10.2.3 + '@fireproof/vendor': + specifier: ^1.0.0 + version: 1.0.0 '@ipld/unixfs': specifier: ^3.0.0 version: 3.0.0 - '@web3-storage/pail': - specifier: ^0.6.0 - version: 0.6.0 - cborg: - specifier: ^4.2.6 - version: 4.2.6 charwise: specifier: ^3.0.1 version: 3.0.1 @@ -41,9 +29,6 @@ importers: idb: specifier: ^8.0.0 version: 8.0.0 - ipfs-unixfs-exporter: - specifier: ^13.6.1 - version: 13.6.1 memfs: specifier: ^4.14.1 version: 4.14.1 @@ -661,6 +646,9 @@ packages: resolution: {integrity: sha512-2b/g5hRmpbb1o4GnTZax9N9m0FXzz9OV42ZzI4rDDMDuHUqigAiQCEWChBWCY4ztAGVRjoWT19v0yMmc5/L5kA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@fireproof/vendor@1.0.0': + resolution: {integrity: sha512-ucN0ICj7xbpbO3NWjAKKjMHkBn4vSRUdKn14LLaA0MILJowty+H4XBB8dCobwKemhW49ha5p520NOLRzJSKsow==} + '@humanfs/core@0.19.1': resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} engines: {node: '>=18.18.0'} @@ -701,18 +689,6 @@ packages: peerDependencies: '@types/node': '>=18' - '@ipld/car@5.3.3': - resolution: {integrity: sha512-4vgV5Ml4HCJ2iTx7vYhu0ui+Xxo1HQTtVeYgD+JKd5Wij8TlOFZnxOSickqpLcuf1fdGEStgqVItx15UWfzDYA==} - engines: {node: '>=16.0.0', npm: '>=7.0.0'} - - '@ipld/dag-cbor@9.2.2': - resolution: {integrity: sha512-uIEOuruCqKTP50OBWwgz4Js2+LhiBQaxc57cnP71f45b1mHEAo1OCR1Zn/TbvSW/mV1x+JqhacIktkKyaYqhCw==} - engines: {node: '>=16.0.0', npm: '>=7.0.0'} - - '@ipld/dag-json@10.2.3': - resolution: {integrity: sha512-itacv1j1hvYgLox2B42Msn70QLzcr0MEo5yGIENuw2SM/lQzq9bmBiMky+kDsIrsqqblKTXcHBZnnmK7D4a6ZQ==} - engines: {node: '>=16.0.0', npm: '>=7.0.0'} - '@ipld/dag-pb@4.0.8': resolution: {integrity: sha512-693AqMY2jvhe+w4jSwjnDrbhxIu39gm1H4f6/KD5gG+6VFMM6EXV7vq85BvEf8CRsnA0+auWfA29/S8gbWI0Ew==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} @@ -1167,10 +1143,6 @@ packages: resolution: {integrity: sha512-rKm5FXkpsCyeqh8tdirtRUHvgNytWNMiaVKdctsvKOJvqnDVPAAQcz9Wmgo7bSwoLwtSHcDaRoxY7olV7J4QnA==} engines: {node: '>=18.20.0'} - '@web3-storage/pail@0.6.0': - resolution: {integrity: sha512-/lIBu8pR92AEcCqYqjcxc3kUSLLRGQLomUes7PAyoK4lNe4ttm8Wr7WFcrbJmWTBJZmYE6npbYcVio8U+OexpA==} - hasBin: true - '@zip.js/zip.js@2.7.52': resolution: {integrity: sha512-+5g7FQswvrCHwYKNMd/KFxZSObctLSsQOgqBSi0LzwHo3li9Eh1w5cF5ndjQw9Zbr3ajVnd2+XyiX85gAetx1Q==} engines: {bun: '>=0.7.0', deno: '>=1.0.0', node: '>=16.5.0'} @@ -1330,10 +1302,6 @@ packages: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} - cborg@4.2.6: - resolution: {integrity: sha512-77vo4KlSwfjCIXcyZUVei4l2gdjesSCeYSx4U/Upwix7pcWZq8uw21sVRpjwn7mjEi//ieJPTj1MRWDHmud1Rg==} - hasBin: true - chai@5.1.2: resolution: {integrity: sha512-aGtmf24DW6MLHHG5gCx4zaI3uBq3KRtxeVs0DjFH6Z0rDNbsvTxFASFvdj79pxjxZ8/5u3PIiN3IwEIQkiiuPw==} engines: {node: '>=12'} @@ -1877,9 +1845,6 @@ packages: resolution: {integrity: sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==} engines: {node: '>= 12'} - ipfs-unixfs-exporter@13.6.1: - resolution: {integrity: sha512-pYPI4oBTWao2//sFzAL0pURyojn79q/u5BuK6L5/nVbVUQVw6DcVP5uB1ySdWlTM2H+0Zlhp9+OL9aJBRIICpg==} - ipfs-unixfs@11.1.2: resolution: {integrity: sha512-HVjrACOhU8RgMskcrfydk+FDAE9pFKr8tneKLaVYQ2f81HUKXoiSdgsAJY/jt7Ieyj4tE12TZGduIeWtNpScOw==} engines: {node: '>=16.0.0', npm: '>=7.0.0'} @@ -2204,10 +2169,6 @@ packages: outvariant@1.4.3: resolution: {integrity: sha512-+Sl2UErvtsoajRDKCE5/dBz4DIvHXQQnAxtQTF04OJxY0+DyZXSo5P5Bb7XYWOh81syohlYL24hbDwxedPUJCA==} - p-defer@4.0.0: - resolution: {integrity: sha512-Vb3QRvQ0Y5XnF40ZUWW7JfLogicVh/EnA5gBIvKDJoYpeI82+1E3AlB9yOcKFS0AhHrWVnAQO39fbR0G99IVEQ==} - engines: {node: '>=12'} - p-defer@4.0.1: resolution: {integrity: sha512-Mr5KC5efvAK5VUptYEIopP1bakB85k2IWXaRC0rsh1uwn1L6M0LVml8OIQ4Gudg4oyZakf7FmeRLkMMtZW1i5A==} engines: {node: '>=12'} @@ -3419,6 +3380,24 @@ snapshots: dependencies: levn: 0.4.1 + '@fireproof/vendor@1.0.0': + dependencies: + '@ipld/dag-pb': 4.1.2 + '@multiformats/murmur3': 2.1.8 + hamt-sharding: 3.0.6 + interface-blockstore: 5.3.1 + ipfs-unixfs: 11.1.2 + it-filter: 3.1.1 + it-last: 3.0.6 + it-map: 3.1.1 + it-parallel: 3.0.8 + it-pipe: 3.0.1 + it-pushable: 3.2.3 + multiformats: 13.3.1 + p-queue: 8.0.1 + progress-events: 1.0.1 + varint: 6.0.0 + '@humanfs/core@0.19.1': {} '@humanfs/node@0.16.6': @@ -3458,23 +3437,6 @@ snapshots: dependencies: '@types/node': 22.10.1 - '@ipld/car@5.3.3': - dependencies: - '@ipld/dag-cbor': 9.2.2 - cborg: 4.2.6 - multiformats: 13.3.1 - varint: 6.0.0 - - '@ipld/dag-cbor@9.2.2': - dependencies: - cborg: 4.2.6 - multiformats: 13.3.1 - - '@ipld/dag-json@10.2.3': - dependencies: - cborg: 4.2.6 - multiformats: 13.3.1 - '@ipld/dag-pb@4.0.8': dependencies: multiformats: 13.3.1 @@ -4000,11 +3962,6 @@ snapshots: transitivePeerDependencies: - supports-color - '@web3-storage/pail@0.6.0': - dependencies: - '@ipld/dag-cbor': 9.2.2 - multiformats: 13.3.1 - '@zip.js/zip.js@2.7.52': {} abort-controller@3.0.0: @@ -4165,8 +4122,6 @@ snapshots: callsites@3.1.0: {} - cborg@4.2.6: {} - chai@5.1.2: dependencies: assertion-error: 2.0.1 @@ -4798,25 +4753,6 @@ snapshots: jsbn: 1.1.0 sprintf-js: 1.1.3 - ipfs-unixfs-exporter@13.6.1: - dependencies: - '@ipld/dag-cbor': 9.2.2 - '@ipld/dag-json': 10.2.3 - '@ipld/dag-pb': 4.1.2 - '@multiformats/murmur3': 2.1.8 - hamt-sharding: 3.0.6 - interface-blockstore: 5.3.1 - ipfs-unixfs: 11.1.2 - it-filter: 3.1.1 - it-last: 3.0.6 - it-map: 3.1.1 - it-parallel: 3.0.8 - it-pipe: 3.0.1 - it-pushable: 3.2.3 - multiformats: 13.3.1 - p-queue: 8.0.1 - progress-events: 1.0.1 - ipfs-unixfs@11.1.2: dependencies: err-code: 3.0.1 @@ -4896,7 +4832,7 @@ snapshots: it-pushable@3.2.3: dependencies: - p-defer: 4.0.0 + p-defer: 4.0.1 it-stream-types@2.0.1: {} @@ -5130,8 +5066,6 @@ snapshots: outvariant@1.4.3: {} - p-defer@4.0.0: {} - p-defer@4.0.1: {} p-limit@3.1.0: diff --git a/smoke/esm/it.sh b/smoke/esm/it.sh index 131a47461..7014ee15f 100644 --- a/smoke/esm/it.sh +++ b/smoke/esm/it.sh @@ -12,13 +12,15 @@ fi $dockerCompose up -d packageDir=../../dist/fireproof-core +user="admin$(date +%s)" token=$(curl \ --retry 10 --retry-max-time 30 --retry-all-errors \ -X PUT \ -H "Content-type: application/json" \ - -d '{ "name": "admin", "password": "admin" }' \ - 'http://localhost:4873/-/user/org.couchdb.user:admin' | jq .token) + -d "{ \"name\": \"$user\", \"password\": \"admin\" }" \ + 'http://localhost:4873/-/user/org.couchdb.user:$user' | jq .token) +echo "Token: $user:$token" cat < $packageDir/.npmrc ; .npmrc enable-pre-post-scripts=true diff --git a/src/blockstore/commitor.ts b/src/blockstore/commitor.ts index b6fa28601..92e612cd2 100644 --- a/src/blockstore/commitor.ts +++ b/src/blockstore/commitor.ts @@ -1,5 +1,5 @@ -import { FileTransactionMeta } from "../types"; -import { CarTransaction } from "./transaction"; +import { FileTransactionMeta } from "../types.js"; +import { CarTransaction } from "./transaction.js"; import { AnyBlock, AnyLink, @@ -14,13 +14,13 @@ import { toCIDBlock, TransactionMeta, WALStore, -} from "./types"; -import * as CBW from "@ipld/car/buffer-writer"; +} from "./types.js"; +import * as CBW from "@fireproof/vendor/@ipld/car/buffer-writer"; import { CID } from "multiformats"; import { encode } from "../runtime/wait-pr-multiformats/block.js"; import { BlockEncoder } from "../runtime/wait-pr-multiformats/codec-interface.js"; import { sha256 as hasher } from "multiformats/hashes/sha2"; -import * as dagCodec from "@ipld/dag-cbor"; +import * as dagCodec from "@fireproof/vendor/@ipld/dag-cbor"; async function encodeCarFile(roots: AnyLink[], t: CarMakeable, codec: BlockEncoder): Promise { let size = 0; diff --git a/src/blockstore/fragment-gateway.ts b/src/blockstore/fragment-gateway.ts index 1d125c79d..2fabf2528 100644 --- a/src/blockstore/fragment-gateway.ts +++ b/src/blockstore/fragment-gateway.ts @@ -1,7 +1,7 @@ import { Logger, Result, URI } from "@adviser/cement"; import { base58btc } from "multiformats/bases/base58"; -import { encode, decode } from "cborg"; +import { encode, decode } from "@fireproof/vendor/cborg"; import { Gateway, GetResult, UnsubscribeResult, VoidResult } from "./gateway.js"; import { SuperThis } from "../types.js"; import { ensureSuperLog } from "../utils.js"; diff --git a/src/blockstore/loader-helpers.ts b/src/blockstore/loader-helpers.ts index 5baac4565..1e03a935f 100644 --- a/src/blockstore/loader-helpers.ts +++ b/src/blockstore/loader-helpers.ts @@ -1,12 +1,12 @@ import { decode } from "../runtime/wait-pr-multiformats/block.js"; import { sha256 as hasher } from "multiformats/hashes/sha2"; -import * as dagCodec from "@ipld/dag-cbor"; +import * as dagCodec from "@fireproof/vendor/@ipld/dag-cbor"; import type { Logger } from "@adviser/cement"; import { CarHeader } from "./types.js"; // import { decodeRunLength } from "../runtime/keyed-crypto.js"; // import { base58btc } from "multiformats/bases/base58"; -import { CarReader } from "@ipld/car/reader"; +import { CarReader } from "@fireproof/vendor/@ipld/car/reader"; // export async function encodeCarHeader(fp: CarHeader) { // return (await encode({ diff --git a/src/blockstore/loader.ts b/src/blockstore/loader.ts index 31007af61..195e3ee73 100644 --- a/src/blockstore/loader.ts +++ b/src/blockstore/loader.ts @@ -1,5 +1,5 @@ import pLimit from "p-limit"; -import { CarReader } from "@ipld/car/reader"; +import { CarReader } from "@fireproof/vendor/@ipld/car/reader"; import { Logger, ResolveOnce } from "@adviser/cement"; // import { uuidv4 } from "uuidv7"; diff --git a/src/blockstore/meta-key-helper.ts b/src/blockstore/meta-key-helper.ts index a7d4c070c..0bded1576 100644 --- a/src/blockstore/meta-key-helper.ts +++ b/src/blockstore/meta-key-helper.ts @@ -1,27 +1,38 @@ -import { format, parse } from "@ipld/dag-json"; -import { EventBlock, decodeEventBlock } from "@web3-storage/pail/clock"; -import { EventView } from "@web3-storage/pail/clock/api"; +import { format, parse } from "@fireproof/vendor/@ipld/dag-json"; +import { EventBlock, decodeEventBlock } from "@fireproof/vendor/@web3-storage/pail/clock"; +import { EventView } from "@fireproof/vendor/@web3-storage/pail/clock/api"; import type { DbMeta, CarClockHead, DbMetaEventBlock, CarClockLink } from "./types.js"; import { SuperThis, CRDTEntry } from "../types.js"; import { CID, Link } from "multiformats"; import { base64pad } from "multiformats/bases/base64"; import { Result, URI } from "@adviser/cement"; import { getKeyBag } from "../runtime/key-bag.js"; +import { ensureLogger } from "../utils.js"; export async function decodeGatewayMetaBytesToDbMeta(sthis: SuperThis, byteHeads: Uint8Array) { const crdtEntries = JSON.parse(sthis.txt.decode(byteHeads)) as CRDTEntry[]; - if (!crdtEntries.length) { - sthis.logger.Debug().Any("byteHeads", byteHeads).Msg("No CRDT entries found"); + if (!Array.isArray(crdtEntries)) { + sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries"); return []; } - if (!crdtEntries.map) { - sthis.logger.Debug().Str("crdtEntries", JSON.stringify(crdtEntries)).Msg("No data in CRDT entries"); + if (!crdtEntries.length) { + sthis.logger.Debug().Any("byteHeads", byteHeads).Msg("No CRDT entries found"); return []; } + const logger = ensureLogger(sthis, "decodeGatewayMetaBytesToDbMeta"); return Promise.all( crdtEntries.map(async (crdtEntry) => { const eventBlock = await decodeEventBlock<{ dbMeta: Uint8Array }>(base64pad.decode(crdtEntry.data)); const dbMeta = parse(sthis.txt.decode(eventBlock.value.data.dbMeta)); + logger + .Debug() + .Any("crdtEntry", { + crdtEntry, + eventBlock, + dbMeta, + dbMetaStrings: dbMeta.cars.map((car) => car.toString()), + }) + .Msg("CRDT entry"); return { eventCid: eventBlock.cid as CarClockLink, parents: crdtEntry.parents, diff --git a/src/blockstore/store.ts b/src/blockstore/store.ts index b45a1e8c0..308530601 100644 --- a/src/blockstore/store.ts +++ b/src/blockstore/store.ts @@ -1,4 +1,4 @@ -import { format, parse, ToString } from "@ipld/dag-json"; +import { format, parse, ToString } from "@fireproof/vendor/@ipld/dag-json"; import { exception2Result, Logger, ResolveOnce, Result, URI } from "@adviser/cement"; import type { AnyBlock, diff --git a/src/blockstore/task-manager.ts b/src/blockstore/task-manager.ts index 0b1dda2a1..8e7550867 100644 --- a/src/blockstore/task-manager.ts +++ b/src/blockstore/task-manager.ts @@ -10,6 +10,7 @@ interface TaskItem { } export class TaskManager { + // we need to remove the events after some time private readonly eventsWeHandled = new Set(); private queue: TaskItem[] = []; @@ -37,6 +38,7 @@ export class TaskManager { const filteredQueue = this.queue.filter(({ cid }) => !this.eventsWeHandled.has(cid)); const first = filteredQueue[0]; if (!first) { + this.isProcessing = false; return; } try { diff --git a/src/blockstore/transaction.ts b/src/blockstore/transaction.ts index 79bde7326..902f62df4 100644 --- a/src/blockstore/transaction.ts +++ b/src/blockstore/transaction.ts @@ -1,5 +1,5 @@ -import { MemoryBlockstore } from "@web3-storage/pail/block"; -import { BlockFetcher as BlockFetcherApi } from "@web3-storage/pail/api"; +import { MemoryBlockstore } from "@fireproof/vendor/@web3-storage/pail/block"; +import { BlockFetcher as BlockFetcherApi } from "@fireproof/vendor/@web3-storage/pail/api"; import { AnyAnyLink, diff --git a/src/blockstore/types.ts b/src/blockstore/types.ts index 8a1defe7a..d85590018 100644 --- a/src/blockstore/types.ts +++ b/src/blockstore/types.ts @@ -1,13 +1,13 @@ import type { CID, Link, Version } from "multiformats"; -import type { BlockCodec } from "../runtime/wait-pr-multiformats/codec-interface"; +import type { BlockCodec } from "../runtime/wait-pr-multiformats/codec-interface.js"; import { DocFileMeta, Falsy, StoreType, SuperThis } from "../types.js"; import { BlockFetcher, CarTransaction } from "./transaction.js"; import { Logger, Result } from "../utils.js"; import { CommitQueue } from "./commit-queue.js"; import { KeyBagOpts } from "../runtime/key-bag.js"; import { CoerceURI, CryptoRuntime, CTCryptoKey, URI } from "@adviser/cement"; -import { EventBlock } from "@web3-storage/pail/clock"; -import { TaskManager } from "./task-manager"; +import { EventBlock } from "@fireproof/vendor/@web3-storage/pail/clock"; +import { TaskManager } from "./task-manager.js"; export type AnyLink = Link; export type CarGroup = AnyLink[]; diff --git a/src/crdt-clock.ts b/src/crdt-clock.ts index 3f927fcdc..07b499c0d 100644 --- a/src/crdt-clock.ts +++ b/src/crdt-clock.ts @@ -1,5 +1,5 @@ -import { advance } from "@web3-storage/pail/clock"; -import { root } from "@web3-storage/pail/crdt"; +import { advance } from "@fireproof/vendor/@web3-storage/pail/clock"; +import { root } from "@fireproof/vendor/@web3-storage/pail/crdt"; import { Logger, ResolveOnce } from "@adviser/cement"; import { clockChangesSince } from "./crdt-helpers.js"; diff --git a/src/crdt-helpers.ts b/src/crdt-helpers.ts index d2772bcb4..8a3ae33db 100644 --- a/src/crdt-helpers.ts +++ b/src/crdt-helpers.ts @@ -1,11 +1,11 @@ import { encode, decode, Block } from "./runtime/wait-pr-multiformats/block.js"; import { parse } from "multiformats/link"; import { sha256 as hasher } from "multiformats/hashes/sha2"; -import * as codec from "@ipld/dag-cbor"; -import { put, get, entries, root } from "@web3-storage/pail/crdt"; -import { EventBlockView, EventLink, Operation, PutOperation } from "@web3-storage/pail/crdt/api"; -import { EventFetcher, vis } from "@web3-storage/pail/clock"; -import * as Batch from "@web3-storage/pail/crdt/batch"; +import * as codec from "@fireproof/vendor/@ipld/dag-cbor"; +import { put, get, entries, root } from "@fireproof/vendor/@web3-storage/pail/crdt"; +import { EventBlockView, EventLink, Operation, PutOperation } from "@fireproof/vendor/@web3-storage/pail/crdt/api"; +import { EventFetcher, vis } from "@fireproof/vendor/@web3-storage/pail/clock"; +import * as Batch from "@fireproof/vendor/@web3-storage/pail/crdt/batch"; import { type EncryptedBlockstore, CarTransaction, @@ -30,7 +30,7 @@ import { type DocTypes, throwFalsy, } from "./types.js"; -import { Result } from "@web3-storage/pail/crdt/api"; +import { Result } from "@fireproof/vendor/@web3-storage/pail/crdt/api"; import { Logger } from "@adviser/cement"; // eslint-disable-next-line @typescript-eslint/no-unused-vars diff --git a/src/index.ts b/src/index.ts index 5d8d4bb91..10c1117ff 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,7 +8,7 @@ export * from "./indexer.js"; export * as bs from "./blockstore/index.js"; export * as blockstore from "./blockstore/index.js"; -import { type Connectable } from "./blockstore"; +import { type Connectable } from "./blockstore/index.js"; export { Connectable }; export * as rt from "./runtime/index.js"; diff --git a/src/indexer-helpers.ts b/src/indexer-helpers.ts index 90d7b20fc..ff8d60e8b 100644 --- a/src/indexer-helpers.ts +++ b/src/indexer-helpers.ts @@ -1,7 +1,7 @@ import type { Block, Link } from "multiformats"; import { create } from "./runtime/wait-pr-multiformats/block.js"; import { sha256 as hasher } from "multiformats/hashes/sha2"; -import * as codec from "@ipld/dag-cbor"; +import * as codec from "@fireproof/vendor/@ipld/dag-cbor"; // @ts-expect-error "charwise" has no types import charwise from "charwise"; diff --git a/src/react/index.ts b/src/react/index.ts index 3227cc76c..1b07956b8 100644 --- a/src/react/index.ts +++ b/src/react/index.ts @@ -1,7 +1,7 @@ -export { type TLUseDocument, useDocument } from "./useDocument"; -export { FireproofCtx, type UseFireproof, useFireproof } from "./useFireproof"; -export { type TLUseLiveQuery, useLiveQuery } from "./useLiveQuery"; -export { type TLUseAllDocs, useAllDocs } from "./useAllDocs"; -export { type TLUseChanges, useChanges } from "./useChanges"; +export { type TLUseDocument, useDocument } from "./useDocument.js"; +export { FireproofCtx, type UseFireproof, useFireproof } from "./useFireproof.js"; +export { type TLUseLiveQuery, useLiveQuery } from "./useLiveQuery.js"; +export { type TLUseAllDocs, useAllDocs } from "./useAllDocs.js"; +export { type TLUseChanges, useChanges } from "./useChanges.js"; // why is this there is should be a package system // export * from "@fireproof/core"; diff --git a/src/react/useAllDocs.ts b/src/react/useAllDocs.ts index 01fb823fc..626ddb116 100644 --- a/src/react/useAllDocs.ts +++ b/src/react/useAllDocs.ts @@ -1,6 +1,6 @@ import { Database, DocTypes } from "@fireproof/core"; -import { AllDocsResult, useFireproof, UseAllDocs } from "./useFireproof"; +import { AllDocsResult, useFireproof, UseAllDocs } from "./useFireproof.js"; export interface TLUseAllDocs { (...args: Parameters): AllDocsResult; diff --git a/src/react/useChanges.ts b/src/react/useChanges.ts index ca5fe20db..be60c7ac0 100644 --- a/src/react/useChanges.ts +++ b/src/react/useChanges.ts @@ -1,6 +1,6 @@ import { Database, DocTypes } from "@fireproof/core"; -import { ChangesResult, useFireproof, UseChanges } from "./useFireproof"; +import { ChangesResult, useFireproof, UseChanges } from "./useFireproof.js"; export interface TLUseChanges { (...args: Parameters): ChangesResult; diff --git a/src/react/useDocument.ts b/src/react/useDocument.ts index 381e25191..23ae55375 100644 --- a/src/react/useDocument.ts +++ b/src/react/useDocument.ts @@ -1,6 +1,6 @@ import { Database, DocTypes, DocWithId } from "@fireproof/core"; -import { UseDocument, UseDocumentResult, useFireproof } from "./useFireproof"; +import { UseDocument, UseDocumentResult, useFireproof } from "./useFireproof.js"; export interface TLUseDocument { (initialDoc: DocWithId): UseDocumentResult; diff --git a/src/react/useFireproof.ts b/src/react/useFireproof.ts index 042f72f5c..7eec5ff0e 100644 --- a/src/react/useFireproof.ts +++ b/src/react/useFireproof.ts @@ -13,7 +13,7 @@ import type { } from "@fireproof/core"; import { fireproof } from "@fireproof/core"; import { useCallback, useEffect, useMemo, useState } from "react"; -import { AllDocsQueryOpts, ChangesOptions, ClockHead } from "../types"; +import { AllDocsQueryOpts, ChangesOptions, ClockHead } from "../types.js"; export interface LiveQueryResult { readonly docs: DocWithId[]; diff --git a/src/react/useLiveQuery.ts b/src/react/useLiveQuery.ts index 806f1c577..981ff64af 100644 --- a/src/react/useLiveQuery.ts +++ b/src/react/useLiveQuery.ts @@ -1,6 +1,6 @@ import { Database, DocFragment, DocTypes, IndexKeyType } from "@fireproof/core"; -import { LiveQueryResult, useFireproof, UseLiveQuery } from "./useFireproof"; +import { LiveQueryResult, useFireproof, UseLiveQuery } from "./useFireproof.js"; export interface TLUseLiveQuery { ( diff --git a/src/runtime/files.ts b/src/runtime/files.ts index e8e5c0f50..2c23d769b 100644 --- a/src/runtime/files.ts +++ b/src/runtime/files.ts @@ -6,7 +6,7 @@ import { withWidth } from "@ipld/unixfs/file/layout/balanced"; import type { View } from "@ipld/unixfs"; import { DocFileMeta } from "../types.js"; -import { exporter, ReadableStorage } from "ipfs-unixfs-exporter"; +import { exporter, ReadableStorage } from "@fireproof/vendor/ipfs-unixfs-exporter"; import { BlobLike, AnyLink, AnyBlock } from "../blockstore/index.js"; const queuingStrategy = UnixFS.withCapacity(); diff --git a/src/runtime/keyed-crypto.ts b/src/runtime/keyed-crypto.ts index e4ab55e94..0f643af1c 100644 --- a/src/runtime/keyed-crypto.ts +++ b/src/runtime/keyed-crypto.ts @@ -1,11 +1,11 @@ import { CryptoRuntime, Logger, URI } from "@adviser/cement"; -import { BytesWithIv, CodecOpts, IvAndBytes, IvKeyIdData, KeyedCrypto, KeyWithFingerPrint } from "../blockstore"; +import { BytesWithIv, CodecOpts, IvAndBytes, IvKeyIdData, KeyedCrypto, KeyWithFingerPrint } from "../blockstore/index.js"; import { ensureLogger, UInt8ArrayEqual } from "../utils.js"; -import { KeyBag } from "./key-bag"; -import type { BlockCodec } from "./wait-pr-multiformats/codec-interface"; +import { KeyBag } from "./key-bag.js"; +import type { BlockCodec } from "./wait-pr-multiformats/codec-interface.js"; import { base58btc } from "multiformats/bases/base58"; import { sha256 as hasher } from "multiformats/hashes/sha2"; -import * as CBOR from "cborg"; +import * as CBOR from "@fireproof/vendor/cborg"; import { SuperThis } from "../types.js"; interface GenerateIVFn { diff --git a/src/runtime/wait-pr-multiformats/block.ts b/src/runtime/wait-pr-multiformats/block.ts index 29211f9e3..222fb84c0 100644 --- a/src/runtime/wait-pr-multiformats/block.ts +++ b/src/runtime/wait-pr-multiformats/block.ts @@ -3,7 +3,7 @@ // https://github.com/multiformats/js-multiformats/pull/305 import { bytes as binary, CID, MultihashHasher, BlockView, ByteView, Version, Link } from "multiformats"; import { Block as mfBlock } from "multiformats/block"; -import { BlockDecoder, BlockEncoder } from "./codec-interface"; +import { BlockDecoder, BlockEncoder } from "./codec-interface.js"; // export type Block = mfBlock diff --git a/src/types.ts b/src/types.ts index 474dd70d6..8d6ec5348 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,5 @@ -import type { EventLink } from "@web3-storage/pail/clock/api"; -import type { Operation } from "@web3-storage/pail/crdt/api"; +import type { EventLink } from "@fireproof/vendor/@web3-storage/pail/clock/api"; +import type { Operation } from "@fireproof/vendor/@web3-storage/pail/crdt/api"; import type { DbMeta, StoreOpts, AnyLink } from "./blockstore/index.js"; import { EnvFactoryOpts, Env, Logger, CryptoRuntime } from "@adviser/cement"; diff --git a/src/utils.ts b/src/utils.ts index 92ac67846..6aa59f58f 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -15,7 +15,7 @@ import { JSONFormatter, YAMLFormatter, } from "@adviser/cement"; -import { PathOps, StoreType, SuperThis, SuperThisOpts, TextEndeCoder } from "./types"; +import { PathOps, StoreType, SuperThis, SuperThisOpts, TextEndeCoder } from "./types.js"; import { base58btc } from "multiformats/bases/base58"; export type { Logger }; diff --git a/tests/blockstore/fragment-gateway.test.ts b/tests/blockstore/fragment-gateway.test.ts index ca51f1436..1b7696f5b 100644 --- a/tests/blockstore/fragment-gateway.test.ts +++ b/tests/blockstore/fragment-gateway.test.ts @@ -1,5 +1,5 @@ import { Result, URI } from "@adviser/cement"; -import { mockSuperThis } from "../helpers"; +import { mockSuperThis } from "../helpers.js"; import { bs } from "@fireproof/core"; class TraceGateway implements bs.Gateway { diff --git a/tests/blockstore/keyed-crypto.test.ts b/tests/blockstore/keyed-crypto.test.ts index 0ee361ce9..b0642343b 100644 --- a/tests/blockstore/keyed-crypto.test.ts +++ b/tests/blockstore/keyed-crypto.test.ts @@ -2,8 +2,8 @@ import { bs, rt } from "@fireproof/core"; import { BuildURI, Logger, MockLogger, runtimeFn, toCryptoRuntime, URI } from "@adviser/cement"; import { base58btc } from "multiformats/bases/base58"; import { sha256 as hasher } from "multiformats/hashes/sha2"; -import * as dagCodec from "@ipld/dag-cbor"; -import { MockSuperThis, mockSuperThis } from "../helpers"; +import * as dagCodec from "@fireproof/vendor/@ipld/dag-cbor"; +import { MockSuperThis, mockSuperThis } from "../helpers.js"; describe("KeyBag", () => { let url: URI; diff --git a/tests/blockstore/loader.test.ts b/tests/blockstore/loader.test.ts index c4fd8d441..a0a8b6c59 100644 --- a/tests/blockstore/loader.test.ts +++ b/tests/blockstore/loader.test.ts @@ -1,10 +1,10 @@ -import * as codec from "@ipld/dag-cbor"; +import * as codec from "@fireproof/vendor/@ipld/dag-cbor"; import { sha256 as hasher } from "multiformats/hashes/sha2"; import { BlockView } from "multiformats"; import { CID } from "multiformats/cid"; -import { MemoryBlockstore } from "@web3-storage/pail/block"; +import { MemoryBlockstore } from "@fireproof/vendor/@web3-storage/pail/block"; import { CRDTMeta, IndexTransactionMeta, SuperThis, bs, rt } from "@fireproof/core"; -import { mockSuperThis } from "../helpers"; +import { mockSuperThis } from "../helpers.js"; class MyMemoryBlockStore extends bs.EncryptedBlockstore { readonly memblock = new MemoryBlockstore(); diff --git a/tests/blockstore/store.test.ts b/tests/blockstore/store.test.ts index b784011d1..a2cc04f81 100644 --- a/tests/blockstore/store.test.ts +++ b/tests/blockstore/store.test.ts @@ -1,6 +1,6 @@ import { CID } from "multiformats"; import { bs, NotFoundError, SuperThis } from "@fireproof/core"; -import { mockSuperThis } from "../helpers"; +import { mockSuperThis } from "../helpers.js"; function runtime(sthis: SuperThis) { return bs.toStoreRuntime({}, sthis); diff --git a/tests/blockstore/transaction.test.ts b/tests/blockstore/transaction.test.ts index 0aa0ae7f9..2f5052ee1 100644 --- a/tests/blockstore/transaction.test.ts +++ b/tests/blockstore/transaction.test.ts @@ -1,6 +1,6 @@ import { CID } from "multiformats"; import { bs, SuperThis } from "@fireproof/core"; -import { mockSuperThis } from "../helpers"; +import { mockSuperThis } from "../helpers.js"; describe("Fresh TransactionBlockstore", function () { let blocks: bs.BaseBlockstore; diff --git a/tests/fireproof/config.test.ts b/tests/fireproof/config.test.ts index dab368356..e6a9a363e 100644 --- a/tests/fireproof/config.test.ts +++ b/tests/fireproof/config.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect, beforeAll } from "vitest"; import { dataDir, fireproof, rt } from "@fireproof/core"; import { runtimeFn, URI } from "@adviser/cement"; -import { mockSuperThis } from "../helpers"; +import { mockSuperThis } from "../helpers.js"; describe("runtime", () => { it("runtime", () => { diff --git a/tests/fireproof/crdt.test.ts b/tests/fireproof/crdt.test.ts index ab87bcd12..a0923ebae 100644 --- a/tests/fireproof/crdt.test.ts +++ b/tests/fireproof/crdt.test.ts @@ -2,7 +2,7 @@ import { CRDT } from "@fireproof/core"; import { bs } from "@fireproof/core"; import { CRDTMeta, DocValue } from "@fireproof/core"; import { Index, index } from "@fireproof/core"; -import { mockSuperThis } from "../helpers"; +import { mockSuperThis } from "../helpers.js"; describe("Fresh crdt", function () { let crdt: CRDT<{ hello: string } | { points: number }>; diff --git a/tests/fireproof/indexer.test.ts b/tests/fireproof/indexer.test.ts index c81cc634c..72da6408a 100644 --- a/tests/fireproof/indexer.test.ts +++ b/tests/fireproof/indexer.test.ts @@ -1,5 +1,5 @@ import { Index, index, Database, CRDT, IndexRows } from "@fireproof/core"; -import { mockSuperThis } from "../helpers"; +import { mockSuperThis } from "../helpers.js"; interface TestType { readonly title: string; diff --git a/tests/fireproof/utils.test.ts b/tests/fireproof/utils.test.ts index a9cc162da..ebcd4b4fb 100644 --- a/tests/fireproof/utils.test.ts +++ b/tests/fireproof/utils.test.ts @@ -1,6 +1,6 @@ import { URI } from "@adviser/cement"; import { rt, getStore, ensureSuperLog } from "@fireproof/core"; -import { mockSuperThis } from "../helpers"; +import { mockSuperThis } from "../helpers.js"; // only for test import { UUID } from "uuidv7"; diff --git a/tsconfig.json b/tsconfig.json index d22e6c27d..8c8c64b24 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -25,9 +25,9 @@ // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ /* Modules */ - "module": "commonjs" /* Specify what module code is generated. */, + "module": "nodenext" /* Specify what module code is generated. */, // "rootDir": "./src" /* Specify the root folder within your source files. */, - // "moduleResolution": "nodenext", /* Specify how TypeScript looks up a file from a given module specifier. */ + "moduleResolution": "nodenext" /* Specify how TypeScript looks up a file from a given module specifier. */, // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ "paths": { "@fireproof/core": ["./src/index.js"], @@ -50,7 +50,7 @@ // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ /* JavaScript Support */ - // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ + "allowJs": true /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */, // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ diff --git a/tsup.config.ts b/tsup.config.ts index 3811e817f..a663bd2aa 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -20,14 +20,17 @@ const external = [ "node:url", "assert", "stream", - "better-sqlite3", ]; function skipper(suffix: string[], target: string) { function intercept(build) { const filter = new RegExp(`(${suffix.join("|")})`); build.onResolve({ filter }, async (args) => { - // console.log("skipper", args.path); + if (target.startsWith("/")) { + return build.resolve(target, { kind: args.kind, resolveDir: `/Users/menabe/Software/fproof/fireproof` }); + } else { + return build.resolve(target, { kind: args.kind, resolveDir: args.resolveDir }); + } // const external = Boolean(build.initialOptions.external?.includes(args.path)); // if (external) { // return { path: args.path, external }; @@ -35,12 +38,10 @@ function skipper(suffix: string[], target: string) { // if (args.resolveDir === '') { // return; // } - - return build.resolve(target, { kind: args.kind, resolveDir: args.resolveDir }); }); } return { - name: "esbuild-resolve", + name: "skipper", setup: (build) => { // for (const moduleName of Object.keys(options)) { intercept(build); @@ -49,23 +50,21 @@ function skipper(suffix: string[], target: string) { }; } -const skipIife = { - // "node-filesystem@skip-iife": "../../../bundle-not-impl.js", - // "mem-filesystem@skip-iife": "../../../bundle-not-impl.js", - // "node:fs/promises": "../../../bundle-not-impl.js", - // "fs/promises": "../../../bundle-not-impl.js", - // "../runtime/store-file.js": "../../bundle-not-impl.js", - // "../runtime/gateways/file/gateway.js": "../bundle-not-impl.js", - // "./mem-filesystem.js": "../../../bundle-not-impl.js", - // "./gateways/file/gateway.js": "../bundle-not-impl.js", - // "./node-sys-container.js": "../bundle-not-impl.js", - // "./key-bag-file.js": "../bundle-not-impl.js", -}; +// const skipIife = { +// // "node-filesystem@skip-iife": "../../../bundle-not-impl.js", +// // "mem-filesystem@skip-iife": "../../../bundle-not-impl.js", +// // "node:fs/promises": "../../../bundle-not-impl.js", +// // "fs/promises": "../../../bundle-not-impl.js", +// // "../runtime/store-file.js": "../../bundle-not-impl.js", +// // "../runtime/gateways/file/gateway.js": "../bundle-not-impl.js", +// // "./mem-filesystem.js": "../../../bundle-not-impl.js", +// // "./gateways/file/gateway.js": "../bundle-not-impl.js", +// // "./node-sys-container.js": "../bundle-not-impl.js", +// // "./key-bag-file.js": "../bundle-not-impl.js", +// }; const skipEsm = {}; -const ourMultiformat = { - // "multiformats/block": `${__dirname}/src/runtime/multiformat/block.ts` -}; +const ourMultiformat = {}; const LIBRARY_BUNDLE_OPTIONS: Options = { format: ["esm", "cjs", "iife"], @@ -91,32 +90,32 @@ function packageVersion() { } const LIBRARY_BUNDLES: readonly Options[] = [ - { - ...LIBRARY_BUNDLE_OPTIONS, - external: [...(LIBRARY_BUNDLE_OPTIONS.external || []), ...nodeInternals, ...webInternals], - format: ["iife"], - name: "@fireproof/core", - entry: ["src/index.ts"], - platform: "browser", - outDir: "dist/fireproof-core", - minify: true, - esbuildPlugins: [ - replace({ - __packageVersion__: packageVersion(), - include: /version/, - }), - skipper([...nodeInternals], `${__dirname}/src/bundle-not-impl.js`), - skipper(["./get-file-system-static.js"], "./get-file-system-dynamic.js"), - // skipper(["./gateway-import-static.js"], "././gateway-import-dynamic.js"), - resolve({ - ...skipIife, - ...ourMultiformat, - }), - ], - dts: { - footer: "declare module '@fireproof/core'", - }, - }, + // { + // ...LIBRARY_BUNDLE_OPTIONS, + // external: [...(LIBRARY_BUNDLE_OPTIONS.external || []), ...nodeInternals, ...webInternals], + // format: ["iife"], + // name: "@fireproof/core", + // entry: ["src/index.ts"], + // platform: "browser", + // outDir: "dist/fireproof-core", + // minify: true, + // esbuildPlugins: [ + // replace({ + // __packageVersion__: packageVersion(), + // include: /version/, + // }), + // skipper([...nodeInternals], `${__dirname}/src/bundle-not-impl.js`), + // skipper(["./get-file-system-static.js"], "./get-file-system-dynamic.js"), + // // skipper(["./gateway-import-static.js"], "././gateway-import-dynamic.js"), + // resolve({ + // ...skipIife, + // ...ourMultiformat, + // }), + // ], + // dts: { + // footer: "declare module '@fireproof/core'", + // }, + // }, { ...LIBRARY_BUNDLE_OPTIONS, format: ["esm", "cjs"], @@ -132,7 +131,6 @@ const LIBRARY_BUNDLES: readonly Options[] = [ }), skipper([...nodeInternals, ...webInternals], `${__dirname}/src/bundle-not-impl.js`), skipper(["./get-file-system-static.js"], "./get-file-system-dynamic.js"), - // skipper(["./gateway-import-static.js"], "././gateway-import-dynamic.js"), resolve({ ...ourMultiformat, }), @@ -174,7 +172,6 @@ const LIBRARY_BUNDLES: readonly Options[] = [ __packageVersion__: packageVersion(), include: /version/, }), - // skipper('@skip-iife', `${__dirname}/src/bundle-not-impl.js`), resolve({ ...ourMultiformat, }), @@ -209,6 +206,7 @@ const LIBRARY_BUNDLES: readonly Options[] = [ ...LIBRARY_BUNDLE_OPTIONS, external: [...(LIBRARY_BUNDLE_OPTIONS.external || []), "@fireproof/core", "@fireproof/core/react"], treeshake: true, + format: ["esm", "cjs"], // minify: true, name: "use-fireproof", entry: ["src/use-fireproof/index.ts"],