Added pagination support

docs/table-of-contents.json

@@ -19,5 +19,11 @@
   "how-to": {
     "title": "How To",
     "pages": [["query-with-comunica", "Query with Comunica"]]
+  },
+  "v2": {
+    "title": "Version 2 (not yet released)",
+    "pages": [
+      ["pagination", "Pagination"]
+    ]
   }
 }

docs/v2/index.md

@@ -0,0 +1,4 @@
+# Version 2 (not yet released)
+
+This section of documentation describes upcoming features of LDkit that has not
+yet been released. You can try them by checking out the GitHub repository.

docs/v2/pagination.md

@@ -0,0 +1,52 @@
+# Pagination
+
+Pagination feature enhances data accessibility by enabling users to efficiently
+navigate through large datasets. The feature integrates seamlessly with the
+existing [Lens](../components/lens) `find` method, allowing for controlled data
+retrieval.
+
+### Specify a range of data to query
+
+```ts
+import { createLens } from "ldkit";
+import { schema } from "ldkit/namespaces";
+
+// Create a schema
+const PersonSchema = {
+  "@type": schema.Person,
+  name: schema.name,
+} as const;
+
+// Create a lens instance
+const Persons = createLens(PersonSchema);
+
+await Persons.find({ take: 10 }); // returns first 10 persons or less
+await Persons.find({ take: 10, skip: 10 }); // returns the next set of 10 persons
+await Persons.find(); // returns the default set of 1000 persons maximum
+```
+
+### Setting global defaults
+
+For consistent pagination behavior across an application, you can define a
+global `take` default in the [Context](../components/context). This eliminates
+the need to repeatedly set take in each find call, streamlining the development
+process. The default value for `take` is 1000.
+
+```ts
+import { type Context, createLens, setDefaultContext } from "ldkit";
+
+const context: Context = {
+  sources: ["https://example.com/sparql"],
+  take: 100, // set the default take value to 100
+};
+
+setDefaultContext(context);
+
+const customContext: Context = {
+  ...context,
+  take: 10,
+};
+
+const First = createLens(FirstSchema); // will use the default context with take = 100
+const Second = createLens(SecondSchema, customContext); // will use custom context with take = 10
+```

examples/basic/main.ts

@@ -22,7 +22,7 @@ const PersonSchema = {
 const Persons = createLens(PersonSchema, context);
 
 // List all persons
-const persons = await Persons.find(undefined, 10);
+const persons = await Persons.find({ take: 10 });
 for (const person of persons) {
   console.log(person.name); // string
   console.log(person.birthDate); // Date

library/lens/lens.ts

@@ -70,8 +70,15 @@ export class Lens<S extends SchemaPrototype, I = SchemaInterface<S>> {
     return this.decode(graph);
   }
 
-  async find(where?: string | RDF.Quad[], limit?: number) {
-    const q = this.queryBuilder.getQuery(where, limit);
+  async find(
+    options: { where?: string | RDF.Quad[]; take?: number; skip?: number } = {},
+  ) {
+    const { where, take, skip } = {
+      take: 1000,
+      skip: 0,
+      ...options,
+    };
+    const q = this.queryBuilder.getQuery(where, take, skip);
     // TODO: console.log(q);
     const graph = await this.engine.queryGraph(q);
     return this.decode(graph);

library/lens/query_builder.ts

@@ -1,4 +1,4 @@
-import type { Property, Schema } from "../schema/mod.ts";
+import { type Schema } from "../schema/mod.ts";
 import { getSchemaProperties } from "../schema/mod.ts";
 import {
   CONSTRUCT,
@@ -13,19 +13,19 @@ import rdf from "../namespaces/rdf.ts";
 
 import { encode } from "../encoder.ts";
 
-import type { Entity } from "./types.ts";
+import { type Entity } from "./types.ts";
 import { QueryHelper } from "./query_helper.ts";
 
 export class QueryBuilder {
   private readonly schema: Schema;
-  private readonly schemaProperties: Record<string, Property>;
   private readonly context: Context;
+  private readonly takeDefault: number;
   private readonly df: RDF.DataFactory;
 
   constructor(schema: Schema, context: Context) {
     this.schema = schema;
-    this.schemaProperties = getSchemaProperties(this.schema);
     this.context = context;
+    this.takeDefault = 1000;
     this.df = new DataFactory();
   }
 
@@ -37,14 +37,6 @@ export class QueryBuilder {
     );
   }
 
-  private getTypesSignature() {
-    return this.df.quad(
-      this.df.variable!("iri"),
-      this.df.namedNode(rdf.type),
-      this.df.variable!("iri_type"),
-    );
-  }
-
   private entitiesToQuads(entities: Entity[]) {
     const quadArrays = entities.map((entity) =>
       encode(entity, this.schema, this.context)
@@ -113,13 +105,13 @@ export class QueryBuilder {
     return SELECT`(count(?iri) as ?count)`.WHERE`${quads}`.build();
   }
 
-  getQuery(where?: string | RDF.Quad[], limit = 1000) {
+  getQuery(where?: string | RDF.Quad[], limit = this.takeDefault, offset = 0) {
     const selectSubQuery = SELECT.DISTINCT`
       ${this.df.variable!("iri")}
     `.WHERE`
       ${this.getShape(false, true)}
       ${where}
-    `.LIMIT(limit).build();
+    `.LIMIT(limit).OFFSET(offset).build();
 
     const query = CONSTRUCT`
       ${this.getResourceSignature()}

library/rdf.ts

@@ -18,6 +18,7 @@ import type {
 export type LDkitContext = {
   graph?: string;
   language?: string;
+  take?: number;
 };
 
 export type Context =

tests/lens.test.ts → tests/lens_common.test.ts

@@ -1,12 +1,6 @@
 import { assert, assertEquals, Comunica, equal } from "./test_deps.ts";
 
-import {
-  createStore,
-  createStoreContext,
-  emptyStore,
-  ttl,
-  x,
-} from "./test_utils.ts";
+import { initStore, ttl, x } from "./test_utils.ts";
 
 import { createLens } from "../library/lens/mod.ts";
 import { rdf, xsd } from "../library/namespaces/mod.ts";
@@ -82,22 +76,11 @@ const Kubrick = createDirector("StanleyKubrick", "Stanley Kubrick");
 const engine = new Comunica();
 const _ = new DataFactory();
 
-const init = () => {
-  const store = createStore();
+export const init = () => {
+  const { store, context, assertStore, empty } = initStore();
   store.addQuads(defaultStoreContent);
-  const context = createStoreContext(store, {
-    sources: [{ type: "rdfjsSource", value: store }],
-  });
   const directors = createLens(Director, context, engine);
   const movies = createLens(Movie, context, engine);
-  const assertStore = (turtle: string) => {
-    const storeQuads = store.getQuads(null, null, null, null);
-    const expectedQuads = ttl(turtle);
-    assertEquals(storeQuads, expectedQuads);
-  };
-  const empty = async () => {
-    await emptyStore(store);
-  };
   return { directors, movies, assertStore, empty };
 };
 
@@ -128,7 +111,7 @@ Deno.test("Resource / Get multiple resources by IRI", async () => {
 Deno.test("Resource / Get resource by string condition", async () => {
   const { directors } = init();
   const condition = `?iri <${x.name}> "Quentin Tarantino" .`;
-  const result = await directors.find(condition);
+  const result = await directors.find({ where: condition });
 
   assertEquals(result.length, 1);
   assertEquals(result[0], Tarantino);
@@ -141,7 +124,7 @@ Deno.test("Resource / Get resource by quad condition", async () => {
     _.namedNode(x.name),
     _.literal("Quentin Tarantino"),
   );
-  const result = await directors.find([condition]);
+  const result = await directors.find({ where: [condition] });
 
   assertEquals(result.length, 1);
   assertEquals(result[0], Tarantino);

tests/lens_pagination.test.ts

@@ -0,0 +1,74 @@
+import { assertEquals, Comunica } from "./test_deps.ts";
+
+import { initStore, x } from "./test_utils.ts";
+
+import { createLens } from "../library/lens/mod.ts";
+import { xsd } from "../library/namespaces/mod.ts";
+import { DataFactory } from "../library/rdf.ts";
+
+import { type SchemaInterface } from "../library/schema/mod.ts";
+
+const engine = new Comunica();
+const _ = new DataFactory();
+
+const Item = {
+  index: x.index,
+} as const;
+
+type ItemType = SchemaInterface<typeof Item>;
+
+const range = (start: number, end: number) =>
+  [...Array(1 + end - start).keys()].map((i) => i + start);
+
+const defaultStoreContent = range(0, 99).map((i) =>
+  _.quad(
+    _.namedNode(x.Item + i),
+    _.namedNode(x.index),
+    _.literal(i.toString(), xsd.integer),
+  )
+);
+
+const assertContainsRange = (items: ItemType[], start: number, end: number) => {
+  const actual = items.map((item) => item.index);
+  const expected = range(start, end).map((it) => it.toString());
+  assertEquals(actual, expected);
+};
+
+const init = () => {
+  const { store, context, assertStore, empty } = initStore();
+  store.addQuads(defaultStoreContent);
+  const items = createLens(Item, context, engine);
+  return { items, assertStore, empty };
+};
+
+Deno.test("Lens / Pagination / Take", async () => {
+  const { items } = init();
+
+  const result1 = await items.find({ take: 1 });
+  assertContainsRange(result1, 0, 0);
+
+  const result10 = await items.find({ take: 10 });
+  assertContainsRange(result10, 0, 9);
+
+  const result100 = await items.find({ take: 100 });
+  assertContainsRange(result100, 0, 99);
+
+  const result1000 = await items.find({ take: 1000 });
+  assertContainsRange(result1000, 0, 99); // extra items should be ignored
+});
+
+Deno.test("Lens / Pagination / Skip", async () => {
+  const { items } = init();
+
+  const result0 = await items.find({ take: 10, skip: 0 });
+  assertContainsRange(result0, 0, 9);
+
+  const result10 = await items.find({ take: 10, skip: 10 });
+  assertContainsRange(result10, 10, 19);
+
+  const result90 = await items.find({ take: 10, skip: 90 });
+  assertContainsRange(result90, 90, 99);
+
+  const result100 = await items.find({ take: 10, skip: 100 });
+  assertEquals(result100, []); // extra items should be ignored
+});

tests/test_utils.ts

@@ -1,3 +1,5 @@
+import { assertEquals } from "./test_deps.ts";
+
 import {
   type Context,
   DataFactory,
@@ -98,3 +100,17 @@ export const emptyStore = (store: N3.Store) => {
     stream.on("end", resolve);
   });
 };
+
+export const initStore = () => {
+  const store = createStore();
+  const context = createStoreContext(store);
+  const assertStore = (turtle: string) => {
+    const storeQuads = store.getQuads(null, null, null, null);
+    const expectedQuads = ttl(turtle);
+    assertEquals(storeQuads, expectedQuads);
+  };
+  const empty = async () => {
+    await emptyStore(store);
+  };
+  return { store, context, assertStore, empty };
+};