diff --git a/bin/cli.ts b/bin/cli.ts index 605d678..4673b9f 100644 --- a/bin/cli.ts +++ b/bin/cli.ts @@ -144,7 +144,26 @@ async function main() { ordered, ); + if (!quiet) { + client.on("description", (info) => { + if (info.extractor.shapesGraph) { + try { + const mermaid = info.extractor.shapesGraph!.toMermaid(info.shape); + console.log("mermaid:"); + console.log(mermaid); + } catch (ex) { + console.log("Failed mermaid extract"); + } + } else { + console.log("No mermaid extracted"); + } + }); + } + if (verbose) { + client.on("relation", (xs) => + console.log("Relation", xs.source, xs.type.value, xs.node), + ); client.on("fragment", () => { console.error("Fragment!"); }); diff --git a/lib/client.ts b/lib/client.ts index f874f65..faafce5 100644 --- a/lib/client.ts +++ b/lib/client.ts @@ -1,5 +1,5 @@ import { Config, intoConfig } from "./config"; -import { Member } from "./page"; +import { Member, Relation } from "./page"; import rdfDereference, { RdfDereferencer } from "rdf-dereference"; import { FileStateFactory, NoStateFactory, StateFactory } from "./state"; import { CBDShapeExtractor } from "extract-cbd-shape"; @@ -10,7 +10,6 @@ import { Quad_Object, Term } from "@rdfjs/types"; import { enhanced_fetch, extractMainNodeShape, - FetchConfig, getObjects, ModulatorFactory, Notifier, @@ -24,7 +23,7 @@ import debug from "debug"; import type { Writer } from "@ajuvercr/js-runner"; export { intoConfig } from "./config"; -export { retry_fetch, extractMainNodeShape } from "./utils"; +export { extractMainNodeShape, retry_fetch } from "./utils"; export type { Member, Page, Relation } from "./page"; export type { Config, ShapeConfig } from "./config"; @@ -66,7 +65,7 @@ async function getInfo( const resp = await rdfDereference.dereference(config.shapeFile, { localFiles: true, - fetch: config.fetch + fetch: config.fetch, }); const quads = await streamToArray(resp.data); config.shape = { @@ -113,7 +112,7 @@ async function getInfo( timestampPaths.length, isVersionOfPaths.length, ); - } catch (ex: any) { } + } catch (ex: any) {} } if (shapeIds.length > 1) { @@ -164,6 +163,8 @@ type EventKey = string & keyof T; type EventReceiver = (params: T) => void; export type ClientEvents = { + relation: Relation; + description: LDESInfo; fragment: void; mutable: void; poll: void; @@ -297,6 +298,7 @@ export class Client { ); const notifier: Notifier = { + relation: (event) => this.emit("relation", event), error: (ex: any) => this.emit("error", ex), fragment: () => this.emit("fragment", undefined), member: (m) => { @@ -336,25 +338,26 @@ export class Client { this.strategy = this.ordered !== "none" ? new OrderedStrategy( - this.memberManager, - this.fetcher, - notifier, - factory, - this.ordered, - this.config.polling, - this.config.pollInterval, - ) + this.memberManager, + this.fetcher, + notifier, + factory, + this.ordered, + this.config.polling, + this.config.pollInterval, + ) : new UnorderedStrategy( - this.memberManager, - this.fetcher, - notifier, - factory, - this.config.polling, - this.config.pollInterval, - ); + this.memberManager, + this.fetcher, + notifier, + factory, + this.config.polling, + this.config.pollInterval, + ); logger("Found %d views, choosing %s", viewQuads.length, ldesId.value); this.strategy.start(ldesId.value); + this.emit("description", info); } stream(strategy?: { @@ -523,4 +526,3 @@ export async function processor( } }; } - diff --git a/lib/strategy/index.ts b/lib/strategy/index.ts index 99e6eba..1ce9054 100644 --- a/lib/strategy/index.ts +++ b/lib/strategy/index.ts @@ -1,4 +1,4 @@ -import { Member } from "../page"; +import { Member, Relation } from "../page"; import { FetchedPage } from "../pageFetcher"; import { RelationChain } from "../relation"; import { TREE } from "@treecg/types"; @@ -28,8 +28,9 @@ export type PageAndRelation = { }; export type StrategyEvents = { + relation: Relation; member: Member; - fragment: {}; + fragment: { url: string }; mutable: {}; pollCycle: {}; close: {}; diff --git a/lib/strategy/ordered.ts b/lib/strategy/ordered.ts index d0fddd3..09abed5 100644 --- a/lib/strategy/ordered.ts +++ b/lib/strategy/ordered.ts @@ -95,6 +95,7 @@ export class OrderedStrategy { this.handleFetched(page, chain); }, relationFound: ({ from, target }, { chain }) => { + this.notifier.relation(target, {}); from.expected.push(target.node); logger("Relation found %s", target.node); const newChain = chain.push(target.node, this.extractRelation(target)); @@ -119,7 +120,7 @@ export class OrderedStrategy { logger("Member done %s", rel.target); const found = this.findOrDefault(rel); found.extracting -= 1; - this.notifier.fragment({}, {}); + this.notifier.fragment({ url: rel.target }, {}); this.checkEmit(); }, extracted: (member) => { diff --git a/lib/strategy/unordered.ts b/lib/strategy/unordered.ts index 6552d63..d80125f 100644 --- a/lib/strategy/unordered.ts +++ b/lib/strategy/unordered.ts @@ -16,7 +16,7 @@ export class UnorderedStrategy { private inFlight: number = 0; private fetchNotifier: Notifier; - private memberNotifier: Notifier; + private memberNotifier: Notifier; private modulator: Modulator; @@ -61,6 +61,7 @@ export class UnorderedStrategy { this.handleFetched(page, index); }, relationFound: ({ from, target }) => { + this.notifier.relation(target, {}); from.expected.push(target.node); this.inFlight += 1; this.modulator.push({ target: target.node, expected: [from.target] }); @@ -75,11 +76,11 @@ export class UnorderedStrategy { error: (error) => { this.notifier.error(error, {}); }, - done: () => { + done: (_, url) => { memberLogger("Members on page done"); this.inFlight -= 1; this.checkEnd(); - this.notifier.fragment({}, {}); + this.notifier.fragment(url, {}); }, extracted: (mem) => this.notifier.member(mem, {}), }; @@ -108,7 +109,7 @@ export class UnorderedStrategy { private handleFetched(page: FetchedPage, index: number) { this.modulator.finished(index); - this.manager.extractMembers(page, {}, this.memberNotifier); + this.manager.extractMembers(page, { url: page.url }, this.memberNotifier); } private checkEnd() { diff --git a/package-lock.json b/package-lock.json index f446d3a..db5ef96 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,19 +1,19 @@ { "name": "ldes-client", - "version": "0.0.9", + "version": "0.0.9-pr.33.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ldes-client", - "version": "0.0.9", + "version": "0.0.9-pr.33.1", "license": "MIT", "dependencies": { "@treecg/types": "^0.4.6", "@types/debug": "^4.1.12", "commander": "^11.1.0", "debug": "^4.3.4", - "extract-cbd-shape": "^0.1.5", + "extract-cbd-shape": "^0.1.6", "heap-js": "^2.3.0", "n3": "^1.17.3", "rdf-data-factory": "^1.1.2", @@ -2545,9 +2545,9 @@ } }, "node_modules/extract-cbd-shape": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/extract-cbd-shape/-/extract-cbd-shape-0.1.5.tgz", - "integrity": "sha512-nfq69iunKdRU6ElsFTx8vKar/WivrxaZG4lIGCZo9D/Gf8XdCCobyPtXuXN0Jz/VpxS64yL+3/b5c5nlCip5Lw==", + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/extract-cbd-shape/-/extract-cbd-shape-0.1.6.tgz", + "integrity": "sha512-WX/P3Q02fnO/mWtTIYU2snl50E2nMHn+9I+quO2WFsP2pMu4oqCENGw7NrMJj8XK/iYUmbvachDezv5WKPMruQ==", "dependencies": { "@treecg/types": "^0.4.5", "debug": "^4.3.4", diff --git a/package.json b/package.json index 22f4fcf..c56937e 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "ldes-client", "description": "This package provides common tooling to work with LDESes.", - "version": "0.0.9", + "version": "0.0.9-pr.33.1", "main": "dist/lib/client.js", "bin": { "ldes-client": "dist/bin/cli.js" @@ -27,7 +27,7 @@ "@types/debug": "^4.1.12", "commander": "^11.1.0", "debug": "^4.3.4", - "extract-cbd-shape": "^0.1.5", + "extract-cbd-shape": "^0.1.6", "heap-js": "^2.3.0", "n3": "^1.17.3", "rdf-data-factory": "^1.1.2",