From 085c16f4d02df6ea9bf6b0ac4c5ab72aeed69615 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 12 Oct 2024 21:23:13 +0000 Subject: [PATCH 01/16] fix #7642 -- remove qdrant entirely and deprecate centralized semantic search - we will use a more distributed much more lightweight approach --- src/packages/database/package.json | 2 - src/packages/database/qdrant/index.ts | 255 ------------------ src/packages/database/qdrant/jsonl.ts | 151 ----------- src/packages/database/qdrant/sqlite.ts | 100 ------- src/packages/frontend/project/search/body.tsx | 23 +- src/packages/hub/client.coffee | 38 +-- .../pages/api/v2/openai/embeddings/remove.ts | 22 -- .../pages/api/v2/openai/embeddings/save.ts | 22 -- .../pages/api/v2/openai/embeddings/search.ts | 22 -- src/packages/pnpm-lock.yaml | 32 +-- src/packages/server/llm/embeddings-abuse.ts | 97 ------- src/packages/server/llm/embeddings-api.ts | 248 ----------------- src/packages/server/llm/embeddings.ts | 243 ----------------- src/packages/util/db-schema/site-defaults.ts | 2 +- .../util/db-schema/site-settings-extras.ts | 6 +- 15 files changed, 23 insertions(+), 1240 deletions(-) delete mode 100644 src/packages/database/qdrant/index.ts delete mode 100644 src/packages/database/qdrant/jsonl.ts delete mode 100644 src/packages/database/qdrant/sqlite.ts delete mode 100644 src/packages/next/pages/api/v2/openai/embeddings/remove.ts delete mode 100644 src/packages/next/pages/api/v2/openai/embeddings/save.ts delete mode 100644 src/packages/next/pages/api/v2/openai/embeddings/search.ts delete mode 100644 src/packages/server/llm/embeddings-abuse.ts delete mode 100644 src/packages/server/llm/embeddings-api.ts delete mode 100644 src/packages/server/llm/embeddings.ts diff --git a/src/packages/database/package.json b/src/packages/database/package.json index ee0e668937..189cb78a83 100644 --- a/src/packages/database/package.json +++ b/src/packages/database/package.json @@ -7,7 +7,6 @@ "./accounts/*": "./dist/accounts/*.js", "./pool": "./dist/pool/index.js", "./pool/*": "./dist/pool/*.js", - "./qdrant": "./dist/qdrant/index.js", "./postgres/*": "./dist/postgres/*.js", "./postgres/schema": "./dist/postgres/schema/index.js", "./postgres/schema/*": "./dist/postgres/schema/*.js", @@ -21,7 +20,6 @@ "@cocalc/backend": "workspace:*", "@cocalc/database": "workspace:*", "@cocalc/util": "workspace:*", - "@qdrant/js-client-rest": "^1.6.0", "@types/lodash": "^4.14.202", "@types/pg": "^8.6.1", "@types/uuid": "^8.3.1", diff --git a/src/packages/database/qdrant/index.ts b/src/packages/database/qdrant/index.ts deleted file mode 100644 index 665163da4d..0000000000 --- a/src/packages/database/qdrant/index.ts +++ /dev/null @@ -1,255 +0,0 @@ -// NOTE/TODO: there is a grpc client that is faster, but it is "work in progress", -// so we're waiting and will switch later. -// See https://github.com/qdrant/qdrant-js/blob/master/packages/js-client-rest/src/qdrant-client.ts -import { QdrantClient } from "@qdrant/js-client-rest"; -import { getServerSettings } from "@cocalc/database/settings/server-settings"; -import { promises as fs } from "fs"; -import { getLogger } from "@cocalc/backend/logger"; -const log = getLogger("database:qdrant"); -import * as jsonl from "./jsonl"; -import * as sqlite from "./sqlite"; - -export const COLLECTION_NAME = "cocalc"; -const SIZE = 1536; // that's for the openai embeddings api - -async function getAuth(): Promise<{ url: string; apiKey?: string }> { - let { - neural_search_enabled, - kucalc, - qdrant_cluster_url: url, - qdrant_api_key: apiKey, - } = await getServerSettings(); - - if (!neural_search_enabled) { - log.debug("getClient - not enabled"); - throw Error("Qdrant neural search is not enabled."); - } - if (!url && !apiKey && kucalc) { - // There is special case fallback config on kucalc. If the - // directory /secrets/qdrant/qdrant exists *AND* no apiKey is set, - // then the api key is the contents of that file (trimmed), - // and the server has hostname "qdrant". - // We only do this check on kucalc and when the api key and url - // are not set, because of course you can also use an external - // server (e.g., https://cloud.qdrant.io/) even with kucalc. - // This is all so you don't have to find and copy/paste the - // autogenerated kucalc api key. - apiKey = await kucalcApiKey(); - log.debug("getClient - kucalc apiKey defined:", apiKey.length > 0); - if (apiKey) { - url = "http://qdrant:6333"; - } - } - - if (!url) { - throw Error("Qdrant Cluster URL not configured"); - } - - // We polyfill fetch so cocalc still works with node 16. With node 18 this isn't needed. - // Node 16 is end-of-life soon and we will stop supporting it. - if (global.Headers == null) { - log.debug("getClient -- patching in node-fetch"); - const { default: fetch, Headers } = await import("node-fetch"); - global.Headers = Headers; - global.fetch = fetch; - } - - return { url, apiKey }; -} - -const clientCache: { [key: string]: QdrantClient } = {}; -export async function getClient(): Promise { - const { url, apiKey } = await getAuth(); - - const key = `${url}-${apiKey}`; - if (clientCache[key]) { - // we return client that matches the configuration in the database. If you change config - // in database, then you get a different client as soon as getServerSettings() updates. - return clientCache[key]; - } - - log.debug("getClient -- using url = ", url); - - // don't necessarily require apiKey to be nontrivial, e.g., not needed locally for dev purposes. - // NOTE: the client seems to do a good job autoreconnecting even if the - // database is stopped and started. - const client = new QdrantClient({ - url, - ...(apiKey ? { apiKey } : undefined), - }); - await init(client); - clientCache[key] = client; - return client; -} - -async function createIndexes(client) { - log.debug("createIndex"); - // It seems fine to just call this frequently. - // There also might not be any way currently to know whether this index exists. - // Note that it was only a few months ago when indexes got added to qdrant! - await client.createPayloadIndex(COLLECTION_NAME, { - field_name: "url", - field_schema: { - type: "text", - tokenizer: "prefix", - min_token_len: 2, - // should be more than enough, since the maximum length of a filename is 255 characters; the url field is - // of the form "\projects/project_id/files/[filename]#fragmentid", so should easily fit in 1000 characters. - max_token_len: 1000, - lowercase: false, - }, - }); -} - -async function createCollection(client) { - log.debug("createCollection"); - // define our schema. - await client.createCollection(COLLECTION_NAME, { - vectors: { - size: SIZE, - distance: "Cosine", // pretty standard to use cosine - }, - // Use quantization to massively reduce memory and space requirements, as explained here: - // see https://qdrant.tech/documentation/quantization/#setting-up-scalar-quantization - quantization_config: { - scalar: { - type: "int8", - quantile: 0.99, - always_ram: false, - }, - }, - }); -} - -async function init(client) { - log.debug("init"); - const { collections } = await client.getCollections(); - const collectionNames = collections.map((collection) => collection.name); - if (!collectionNames.includes(COLLECTION_NAME)) { - await createCollection(client); - } - await createIndexes(client); -} - -export type Payload = - | { [key: string]: unknown } - | Record - | null - | undefined; - -export interface Point { - id: string | number; - vector: number[]; - payload?: Payload; -} - -export async function upsert(data: Point[]) { - log.debug("upsert"); - const client = await getClient(); - await client.upsert(COLLECTION_NAME, { - wait: true, - points: data, - }); -} - -export async function search({ - id, - vector, - limit, - filter, - selector, - offset, -}: { - vector?: number[]; - id?: string | number; - limit: number; - filter?: object; - selector?; - offset?: number; -}) { - log.debug("search; filter=", filter); - const client = await getClient(); - if (id) { - return await client.recommend(COLLECTION_NAME, { - positive: [id], - limit, - filter, - with_payload: selector == null ? true : selector, - offset, - }); - } else if (vector) { - return await client.search(COLLECTION_NAME, { - vector, - limit, - filter, - with_payload: selector == null ? true : selector, - offset, - }); - } else { - throw Error("id or vector must be specified"); - } -} - -export async function scroll({ - limit, - filter, - selector, - offset, -}: { - limit: number; - filter: object; - selector?; - offset?: number | string; -}) { - log.debug("scroll; filter=", filter); - const client = await getClient(); - return await client.scroll(COLLECTION_NAME, { - limit, - filter, - with_payload: selector == null ? true : selector, - offset, - }); -} - -export async function getPoints(opts): Promise { - const client = await getClient(); - const result = await client - .api("points") - .getPoints({ collection_name: COLLECTION_NAME, ...opts }); - return result.data.result; -} - -export async function deletePoints(opts): Promise { - log.debug("deletePoints="); - const client = await getClient(); - const result = await client - .api("points") - .deletePoints({ collection_name: COLLECTION_NAME, ...opts }); - return result.data.result; -} - -// Read the file /secrets/qdrant/qdrant, convert that -// to a string and trim it and return it.: -async function kucalcApiKey(): Promise { - try { - const data = await fs.readFile("/secrets/qdrant/qdrant"); - return data.toString().trim(); - } catch (_err) { - return ""; - } -} - -export async function jsonlSave(collection = COLLECTION_NAME, file?) { - const { url, apiKey } = await getAuth(); - await jsonl.save({ collection, file, apiKey, url }); -} - -export async function jsonlLoad(collection = COLLECTION_NAME, file?) { - const { url, apiKey } = await getAuth(); - await jsonl.load({ collection, file, apiKey, url }); -} - -export async function sqliteSave(collection = COLLECTION_NAME, file?) { - const { url, apiKey } = await getAuth(); - await sqlite.save({ collection, file, apiKey, url }); -} diff --git a/src/packages/database/qdrant/jsonl.ts b/src/packages/database/qdrant/jsonl.ts deleted file mode 100644 index 0fdeb9f508..0000000000 --- a/src/packages/database/qdrant/jsonl.ts +++ /dev/null @@ -1,151 +0,0 @@ -/* -Save and restore the *data* in a collection to/from a jsonl file. -That's json-lines, i.e., https://jsonlines.org/ - -I wrote this because it's faster and "easier to trust" than Qdrant's built in -snapshot functionality for my use case. It actually seems like this is 10x faster, -probably because it discards all the indexing and other internal metadata. -That's OK for our longterm backups though. - -Use json lines directly is also nice since we can make incremental deduped backups -using bup, and also just visibly inspect the data to see it is there. -It's also useful for dumping a collection, changing params, and reading the collection -back in. -*/ - -// NOTE: this file should be usable outside of the rest of the cocalc code. -// In particular, don't move or rename this file, since that would break -// building kucalc's qdrant docker container. - -import * as fs from "fs"; -import * as readline from "readline"; -import { QdrantClient } from "@qdrant/js-client-rest"; - -const log = console.log; -const DEFAULT_BATCH_SIZE = 1000; - -function getClient({ url, apiKey }) { - return new QdrantClient({ - url, - ...(apiKey ? { apiKey } : undefined), - }); -} - -export async function save({ - collection, - batchSize = DEFAULT_BATCH_SIZE, - file, - url, - apiKey, -}: { - file?: string; - collection: string; - batchSize?: number; - url: string; - apiKey?: string; -}) { - const t = Date.now(); - if (file == null) { - file = `${collection}.jsonl`; - } - const client = getClient({ url, apiKey }); - const info = await client.getCollection(collection); - const { vectors_count } = info; - log( - "save: there are", - vectors_count, - "vectors to save in", - collection, - "to", - file, - ); - - // Create a write stream for the output file - const compressedStream = fs.createWriteStream(file); - - // Fetch all points in the collection in blocks, compressing and - // writing them to the output file - let offset: string | undefined = undefined; - for (let n = 0; n < (vectors_count ?? 0); n += batchSize) { - log("save: from ", n, " to ", n + batchSize); - const { points } = await client.scroll(collection, { - limit: batchSize + (offset ? 1 : 0), - with_payload: true, - with_vector: true, - offset, - }); - if (points == null) continue; - if (offset && points[0]?.id == offset) { - // delete first point since it was the offset. - points.shift(); - } - offset = points[points.length - 1].id as string; - for (const point of points) { - const compressedLine = JSON.stringify(point) + "\n"; - compressedStream.write(compressedLine); - } - } - - // Close the write stream when done - compressedStream.end(); - log("Total time:", (Date.now() - t) / 1000, " seconds"); -} - -// Reads the data into the collection from the json file. -// This does not configure the collection in any way or -// delete anything from the collection. -export async function load({ - collection, - batchSize = DEFAULT_BATCH_SIZE, - file, - url, - apiKey, -}: { - file?: string; - collection: string; - batchSize?: number; - url: string; - apiKey?: string; -}) { - const t = Date.now(); - if (file == null) { - file = `${collection}.jsonl`; - } - const client = getClient({ url, apiKey }); - - const rl = readline.createInterface({ - input: fs.createReadStream(file), - crlfDelay: Infinity, - }); - - let points: any[] = []; // use any[] for convenience, replace with real type - let numParsed = 0; - - const upsertPoints = async () => { - await client.upsert(collection, { - wait: true, - points, - }); - numParsed += points.length; - log("load: upserted ", points.length); - points.length = 0; // reset the batch - }; - - for await (const line of rl) { - const point = JSON.parse(line); - points.push(point); - if (points.length >= batchSize) { - // If we've reached a full batch size, process it - // [insert code to write batch to qdrant database here] - log("load: read ", numParsed, " from disk"); - await upsertPoints(); - } - } - // If there are any remaining points in the batch, process them - if (points.length > 0) { - await upsertPoints(); - } - - log("loadFromJson: finished processing ", numParsed); - log("Total time:", (Date.now() - t) / 1000, " seconds"); -} diff --git a/src/packages/database/qdrant/sqlite.ts b/src/packages/database/qdrant/sqlite.ts deleted file mode 100644 index 41ec12ca85..0000000000 --- a/src/packages/database/qdrant/sqlite.ts +++ /dev/null @@ -1,100 +0,0 @@ -/* -I wrote this for exporting data from a collection to sqlite3. -It might be useful, but it's a lot slower and bigger than -exporting to json-lines, so we're probably not going to use -it, except possibly for interactively exploring data (?). - -E.g., you can look at or query the text field of the payload: - -SELECT json_extract(payload, '$.text') as text FROM cocalc; - -NOTE: If you just want to make a backup, use jsonl -- it's much faster. -The size between json and sqlite3 is the same. -*/ - -import Database from "better-sqlite3"; -import { QdrantClient } from "@qdrant/js-client-rest"; - -const log = console.log; -const DEFAULT_BATCH_SIZE = 1000; - -function getClient({ url, apiKey }) { - return new QdrantClient({ - url, - ...(apiKey ? { apiKey } : undefined), - }); -} - -export async function save({ - file, - collection, - batchSize = DEFAULT_BATCH_SIZE, - url, - apiKey, -}: { - file?: string; - collection: string; - batchSize?: number; - url: string; - apiKey?: string; -}) { - const t = Date.now(); - if (file == null) { - file = `${collection}.db`; - } - const client = getClient({ url, apiKey }); - - const info = await client.getCollection(collection); - const { vectors_count } = info; - log("dump: there are ", vectors_count, "vectors to dump in ", collection); - - // Create sqlite database - const db = new Database(file); - db.pragma("journal_mode = WAL"); - // Delete the table collection if it exists - db.exec(`DROP TABLE IF EXISTS ${collection}`); - // Create a table in the sqlite database called collection - // The table should have the following columns: - // - id: a string - // - payload: an arbitrary json object - // - vector: an array of 1536 double precision floats.: - // In sqlite3 the types are very simple though! - db.exec(`CREATE TABLE ${collection} ( - id TEXT, - payload TEXT, - vector BLOB - )`); - - // Fetch all points in the collection in blocks, inserting them - // into our database. - let offset: string | undefined = undefined; - for (let n = 0; n < (vectors_count ?? 0); n += batchSize) { - log("save: from ", n, " to ", n + batchSize); - const { points } = await client.scroll(collection, { - limit: batchSize + (offset ? 1 : 0), - with_payload: true, - with_vector: true, - offset, - }); - if (points == null) continue; - if (offset && points[0]?.id == offset) { - // delete first point since it was the offset. - points.shift(); - } - offset = points[points.length - 1].id as string; - // insert points into the sqlite3 table collection efficiently: - const insertStmt = db.prepare( - `INSERT INTO ${collection} (id, payload, vector) VALUES (?, ?, ?)`, - ); - - db.transaction(() => { - for (const point of points) { - const { id, payload, vector } = point; - const payloadJson = JSON.stringify(payload); - const vectorBuffer = JSON.stringify(vector); - insertStmt.run(id, payloadJson, vectorBuffer); - } - })(); - } - log("Total time:", (Date.now() - t) / 1000, " seconds"); -} diff --git a/src/packages/frontend/project/search/body.tsx b/src/packages/frontend/project/search/body.tsx index 50388b603b..1670de886c 100644 --- a/src/packages/frontend/project/search/body.tsx +++ b/src/packages/frontend/project/search/body.tsx @@ -12,12 +12,10 @@ be in a single namespace somehow...! import { Button, Card, Col, Input, Row, Space, Tag } from "antd"; import { useEffect, useMemo, useState } from "react"; - import { useProjectContext } from "@cocalc/frontend/project/context"; import { Alert, Checkbox, Well } from "@cocalc/frontend/antd-bootstrap"; import { useActions, useTypedRedux } from "@cocalc/frontend/app-framework"; import { - A, Gap, HelpIcon, Icon, @@ -140,14 +138,8 @@ export const ProjectSearchBody: React.FC<{ New
- Neural search using{" "} - - OpenAI Embeddings - {" "} - and Qdrant: search recently - edited files using a neural network similarity algorithm. - Indexed file types: jupyter, tasks, chat, whiteboards, and - slides. + Neural search: jupyter, tasks, + chat, whiteboards, and slides.
)} @@ -211,16 +203,7 @@ export const ProjectSearchBody: React.FC<{ actions?.setState({ neural_search: !neural_search }) } > - Neural search New{" "} - - This novel search uses{" "} - - OpenAI Embeddings - {" "} - and Qdrant. It searches - recently edited files using a neural network similarity algorithm. - Indexed file types: jupyter, tasks, chat, whiteboards, and slides. - + Neural search )} diff --git a/src/packages/hub/client.coffee b/src/packages/hub/client.coffee index e4f8f6fed2..18d5e3775f 100644 --- a/src/packages/hub/client.coffee +++ b/src/packages/hub/client.coffee @@ -40,7 +40,6 @@ db_schema = require('@cocalc/util/db-schema') generateHash = require("@cocalc/server/auth/hash").default; passwordHash = require("@cocalc/backend/auth/password-hash").default; llm = require('@cocalc/server/llm/index'); -embeddings_api = require('@cocalc/server/llm/embeddings-api'); jupyter_execute = require('@cocalc/server/jupyter/execute').execute; jupyter_kernels = require('@cocalc/server/jupyter/kernels').default; create_project = require("@cocalc/server/projects/create").default; @@ -2085,44 +2084,15 @@ class exports.Client extends EventEmitter dbg("failed -- #{err}") @error_to_client(id:mesg.id, error:"#{err}") + # These are deprecated. Not the best approach. mesg_openai_embeddings_search: (mesg) => - dbg = @dbg("mesg_openai_embeddings_search") - dbg(mesg.input) - if not @account_id? - @error_to_client(id:mesg.id, error:"not signed in") - return - try - matches = await embeddings_api.search(account_id:@account_id, scope:mesg.scope, text:mesg.text, filter:mesg.filter, limit:mesg.limit, selector:mesg.selector, offset:mesg.offset) - @push_to_client(message.openai_embeddings_search_response(id:mesg.id, matches:matches)) - catch err - dbg("failed -- #{err}") - @error_to_client(id:mesg.id, error:"#{err}") + @error_to_client(id:mesg.id, error:"openai_embeddings_search is DEPRECATED") mesg_openai_embeddings_save: (mesg) => - dbg = @dbg("mesg_openai_embeddings_save") - dbg() - if not @account_id? - @error_to_client(id:mesg.id, error:"not signed in") - return - try - ids = await embeddings_api.save(account_id:@account_id, data:mesg.data, project_id:mesg.project_id, path:mesg.path) - @push_to_client(message.openai_embeddings_save_response(id:mesg.id, ids:ids)) - catch err - dbg("failed -- #{err}") - @error_to_client(id:mesg.id, error:"#{err}") + @error_to_client(id:mesg.id, error:"openai_embeddings_save is DEPRECATED") mesg_openai_embeddings_remove: (mesg) => - dbg = @dbg("mesg_openai_embeddings_remove") - dbg() - if not @account_id? - @error_to_client(id:mesg.id, error:"not signed in") - return - try - ids = await embeddings_api.remove(account_id:@account_id, data:mesg.data, project_id:mesg.project_id, path:mesg.path) - @push_to_client(message.openai_embeddings_remove_response(id:mesg.id, ids:ids)) - catch err - dbg("failed -- #{err}") - @error_to_client(id:mesg.id, error:"#{err}") + @error_to_client(id:mesg.id, error:"openai_embeddings_remove is DEPRECATED") mesg_jupyter_execute: (mesg) => dbg = @dbg("mesg_jupyter_execute") diff --git a/src/packages/next/pages/api/v2/openai/embeddings/remove.ts b/src/packages/next/pages/api/v2/openai/embeddings/remove.ts deleted file mode 100644 index f003128c42..0000000000 --- a/src/packages/next/pages/api/v2/openai/embeddings/remove.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { remove } from "@cocalc/server/llm/embeddings-api"; -import getAccountId from "lib/account/get-account"; -import getParams from "lib/api/get-params"; - -export default async function handle(req, res) { - try { - throw Error("currently disabled"); - const ids = await doIt(req); - res.json({ ids, success: true }); - } catch (err) { - res.json({ error: `${err.message}` }); - return; - } -} - -async function doIt(req) { - const account_id = await getAccountId(req); - if (!account_id) { - throw Error("must be signed in"); - } - return await remove({ ...getParams(req), account_id } as any); -} diff --git a/src/packages/next/pages/api/v2/openai/embeddings/save.ts b/src/packages/next/pages/api/v2/openai/embeddings/save.ts deleted file mode 100644 index fd77fc2243..0000000000 --- a/src/packages/next/pages/api/v2/openai/embeddings/save.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { save } from "@cocalc/server/llm/embeddings-api"; -import getAccountId from "lib/account/get-account"; -import getParams from "lib/api/get-params"; - -export default async function handle(req, res) { - try { - throw Error("currently disabled"); - const ids = await doIt(req); - res.json({ ids, success: true }); - } catch (err) { - res.json({ error: `${err.message}` }); - return; - } -} - -async function doIt(req) { - const account_id = await getAccountId(req); - if (!account_id) { - throw Error("must be signed in"); - } - return await save({ ...getParams(req), account_id } as any); -} diff --git a/src/packages/next/pages/api/v2/openai/embeddings/search.ts b/src/packages/next/pages/api/v2/openai/embeddings/search.ts deleted file mode 100644 index 96819d0fdd..0000000000 --- a/src/packages/next/pages/api/v2/openai/embeddings/search.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { search } from "@cocalc/server/llm/embeddings-api"; -import getAccountId from "lib/account/get-account"; -import getParams from "lib/api/get-params"; - -export default async function handle(req, res) { - try { - throw Error("currently disabled"); - const matches = await doIt(req); - res.json({ matches, success: true }); - } catch (err) { - res.json({ error: `${err.message}` }); - return; - } -} - -async function doIt(req) { - const account_id = await getAccountId(req); - if (!account_id) { - throw Error("must be signed in"); - } - return await search({ ...getParams(req), account_id } as any); -} diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index 83b153261d..df57b1d8c5 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -166,9 +166,6 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util - '@qdrant/js-client-rest': - specifier: ^1.6.0 - version: 1.11.0(typescript@5.6.3) '@types/lodash': specifier: ^4.14.202 version: 4.17.9 @@ -189,7 +186,7 @@ importers: version: 8.7.0 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@8.1.1) + version: 4.3.7 immutable: specifier: ^4.3.0 version: 4.3.7 @@ -3657,12 +3654,6 @@ packages: '@protobufjs/utf8@1.1.0': resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} - '@qdrant/js-client-rest@1.11.0': - resolution: {integrity: sha512-RzF+HxL8A7bb/uaxU1jVS1a919bb3FCo1giB/D19UT3d50AYl4+4AyklbsjlXpWEHekbNocQAQ016fqT9hSRtQ==} - engines: {node: '>=18.0.0', pnpm: '>=8'} - peerDependencies: - typescript: '>=4.7' - '@qdrant/js-client-rest@1.12.0': resolution: {integrity: sha512-H8VokZq2DYe9yfKG3c7xPNR+Oc5ZvwMUtPEr1wUO4xVi9w5P89MScJaCc9UW8mS5AR+/Y1h2t1YjSxBFPIYT2Q==} engines: {node: '>=18.0.0', pnpm: '>=8'} @@ -12723,7 +12714,8 @@ snapshots: '@eslint/js@8.57.1': {} - '@fastify/busboy@2.1.0': {} + '@fastify/busboy@2.1.0': + optional: true '@formatjs/cli@6.2.12': {} @@ -13917,13 +13909,6 @@ snapshots: '@protobufjs/utf8@1.1.0': {} - '@qdrant/js-client-rest@1.11.0(typescript@5.6.3)': - dependencies: - '@qdrant/openapi-typescript-fetch': 1.2.6 - '@sevinf/maybe': 0.5.0 - typescript: 5.6.3 - undici: 5.28.4 - '@qdrant/js-client-rest@1.12.0(typescript@5.6.3)': dependencies: '@qdrant/openapi-typescript-fetch': 1.2.6 @@ -13932,7 +13917,8 @@ snapshots: undici: 5.28.4 optional: true - '@qdrant/openapi-typescript-fetch@1.2.6': {} + '@qdrant/openapi-typescript-fetch@1.2.6': + optional: true '@rc-component/async-validator@5.0.4': dependencies: @@ -14141,7 +14127,8 @@ snapshots: transitivePeerDependencies: - debug - '@sevinf/maybe@0.5.0': {} + '@sevinf/maybe@0.5.0': + optional: true '@sinclair/typebox@0.27.8': {} @@ -16498,6 +16485,10 @@ snapshots: dependencies: ms: 2.1.2 + debug@4.3.7: + dependencies: + ms: 2.1.3 + debug@4.3.7(supports-color@8.1.1): dependencies: ms: 2.1.3 @@ -23225,6 +23216,7 @@ snapshots: undici@5.28.4: dependencies: '@fastify/busboy': 2.1.0 + optional: true undici@6.20.0: optional: true diff --git a/src/packages/server/llm/embeddings-abuse.ts b/src/packages/server/llm/embeddings-abuse.ts deleted file mode 100644 index 2214f60b15..0000000000 --- a/src/packages/server/llm/embeddings-abuse.ts +++ /dev/null @@ -1,97 +0,0 @@ -/* -Rate limitations to prevent blatant or accidental (e.g., a bug by me) abuse of -the openai embeddings api endpoint. - -- First note that openai has api rate limits posted here: - https://platform.openai.com/account/rate-limits -For the embedding model, the limits are: -3,000 requests per minute -250,000 tokens per minute - -The cost for embeddings is $0.0004 / 1K tokens (see https://openai.com/pricing). -So 250,000 tokens is $0.10, so it seems like the max spend per hour is $6... -which is nearly $5K/month. - -We want to cap the max spend per *free user* at something like $1/month, which is -2.5 million tokens. So if you use 3500 tokens per hour, that would exceed $1/month, -so we can't just cap per hour for this embeddings stuff, since it is very bursty. -We could do 100K/day, since hitting that every day for a month is about $1.50, which -is probably fine... though we do have over 30K active users, and $45K is a lot of money! - -So let's just think about a max across all users first. If want to spend at most $1K/month -on indexing, that's about 1/5 of the above limit, i.e., 50K/minute, or 3 million/hour. - -We'll cap for now at an average cap of about 5 million tokens per hour; this would -keep cost below $2K/month. - -And we will definitely have a "pay for what you use" option to index and search -any amount of content! - -*/ - -import { getServerSettings } from "@cocalc/database/settings/server-settings"; -import getPool from "@cocalc/database/pool"; - -const QUOTA = [ - //{ upTo: 100000, perUser: 1000 }, // FOR TESTING ONLY! - { upTo: 1000000, perUser: 250000 }, - { upTo: 2000000, perUser: 100000 }, - { upTo: 3000000, perUser: 20000 }, - { upTo: 4000000, perUser: 10000 }, - { upTo: 5000000, perUser: 5000 }, -]; - -function perUserQuotaPerHour(global: number): number { - for (const { upTo, perUser } of QUOTA) { - if (global <= upTo) { - return perUser; - } - } - return 0; -} - -export default async function checkForAbuse(account_id: string): Promise { - const { neural_search_enabled } = await getServerSettings(); - if (!neural_search_enabled) { - // ensure that if you explicitly switch off neural search, then all api requests fail quickly, - // even if some frontend browsers haven't got the message (or don't care). - throw Error("Neural search is currently disabled."); - } - const global = await recentUsage({ - cache: "medium", - period: "1 hour", - }); - const user = await recentUsage({ - cache: "short", - period: "1 hour", - account_id, - }); - if (user > perUserQuotaPerHour(global)) { - throw Error( - "There are too many requests to the embeddings API right now. Please try again in a few minutes." - ); - } -} - -async function recentUsage({ - period, - account_id, - cache, -}: { - period: string; - account_id?: string; - cache?: "short" | "medium" | "long"; -}): Promise { - const pool = getPool(cache); - let query, args; - if (account_id) { - query = `SELECT SUM(tokens) AS usage FROM openai_embedding_log WHERE account_id=$1 AND time >= NOW() - INTERVAL '${period}'`; - args = [account_id]; - } else { - query = `SELECT SUM(tokens) AS usage FROM openai_embedding_log WHERE time >= NOW() - INTERVAL '${period}'`; - args = []; - } - const { rows } = await pool.query(query, args); - // console.log("rows = ", rows); - return parseInt(rows[0]?.["usage"] ?? 0); // undefined = no results in above select, -} diff --git a/src/packages/server/llm/embeddings-api.ts b/src/packages/server/llm/embeddings-api.ts deleted file mode 100644 index b435537ef6..0000000000 --- a/src/packages/server/llm/embeddings-api.ts +++ /dev/null @@ -1,248 +0,0 @@ -import isCollaborator from "@cocalc/server/projects/is-collaborator"; -import type { EmbeddingData } from "@cocalc/util/db-schema/llm"; -import { - MAX_REMOVE_LIMIT, - MAX_SAVE_LIMIT, - MAX_SEARCH_LIMIT, - MAX_SEARCH_TEXT, -} from "@cocalc/util/db-schema/llm"; -import { isValidUUID, is_array } from "@cocalc/util/misc"; -import * as embeddings from "./embeddings"; - -function validateSearchParams({ text, filter, limit, selector, offset }) { - if (text != null) { - if (typeof text != "string") { - throw Error("text must be a string"); - } - if (!text.trim()) { - throw Error("text must not be whitespace"); - } - if (text.length > MAX_SEARCH_TEXT) { - // hard limit on size for *search*. - throw Error(`text must be at most ${MAX_SEARCH_TEXT} characters`); - } - } - if (filter != null && typeof filter != "object") { - throw Error("if filter is not null it must be an object"); - } - if (typeof limit != "number") { - throw Error("limit must be a number"); - } - if (limit <= 0 || limit > MAX_SEARCH_LIMIT) { - throw Error(`limit must be a positive number up to ${MAX_SEARCH_LIMIT}`); - } - if (offset != null) { - if (typeof offset == "number") { - if (offset < 0) { - throw Error("offset must be nonnegative integer or uuid"); - } - } else if (typeof offset == "string") { - if (!isValidUUID(offset)) { - throw Error("offset must be nonnegative integer or uuid"); - } - } - if (text != null && typeof offset != "number") { - throw Error("offset must be a number when doing a vector search"); - } - } - if (selector != null) { - if (typeof selector != "object") { - throw Error( - "selector must object of the form { include?: string[]; exclude?: string[] }", - ); - } - } -} - -export async function search({ - account_id, - scope, - text, - limit, - filter: filter0, - selector, - offset, -}: { - account_id: string; - scope: string | string[]; - text?: string; - limit: number; - filter?: object; - selector?: { include?: string[]; exclude?: string[] }; - offset?: number | string; -}): Promise { - const filter = await scopeFilter(account_id, scope, filter0); - validateSearchParams({ text, filter, limit, selector, offset }); - return await embeddings.search({ - text, - limit, - filter, - selector, - offset, - account_id, - }); -} - -// Creates filter object that further restricts input filter to also have the given scope. -// The scope restricts to only things this user is allowed to see. -// It is just an absolute url path (or array of them) to the cocalc server. -// E.g., if it is "projects/10f0e544-313c-4efe-8718-2142ac97ad11/files/cocalc" that means -// the scope is files in the cocalc directory of the project with id projects/10f0e544-313c-4efe-8718-2142ac97ad11 -// If it is "share", that means anything on the share server, etc. (so anybody can read -- we don't include unlisted). -// This throws an error if the scope isn't sufficient, user doesn't exist, -// request for projects they don't collab on, etc. -// NOTE: in the database we always make the first character of payload.url a single backslash, -// so that we can do prefix searches, which don't exist in qdrant. -async function scopeFilter( - account_id: string, - scope: string | string[], - filter: object = {}, -): Promise { - if (typeof scope != "string" && !is_array(scope)) { - throw Error("scope must be a string or string[]"); - } - if (typeof scope == "string") { - scope = [scope]; - } - - const should: any[] = []; - const knownProjects = new Set(); // efficiency hack - for (const s of scope) { - if (typeof s != "string" || !s) { - throw Error("each entry in the scope must be a nonempty string"); - } - if (s.includes("\\")) { - throw Error("scope may not include backslashes"); - } - if (s.startsWith("projects/")) { - // a project -- parse the project_id and confirm access - const v = s.split("/"); - const project_id = v[1]; - if ( - !knownProjects.has(project_id) && - !(await isCollaborator({ project_id, account_id })) - ) { - throw Error( - `must be a collaborator on the project with id '${project_id}'`, - ); - } - knownProjects.add(project_id); - } else if (s == "share" || s == "share/") { - // ok - } else { - throw Error(`scope "${s}" not supported`); - } - // no error above, so this is a prefix search - // TODO [ ]: make appropriate index on url text field. - should.push({ key: "url", match: { text: "\\" + s } }); - } - if (filter["should"]) { - filter["should"] = [...should, ...filter["should"]]; - } else { - filter["should"] = should; - } - return filter; -} - -/* -Prepare data for saving or deleting from database. -- We check that the account_id is a valid collab on project_id. -- We ensure data[i].payload is an object for each i. -- If needsField is true, then we also ensure data[i].field is set and provides a valid field into the payload. -*/ -async function prepareData( - account_id: string, - project_id: string, - path: string, - data: EmbeddingData[], - needsText: boolean, -): Promise { - if (!is_array(data)) { - throw Error("data must be an array"); - } - if (!(await isCollaborator({ account_id, project_id }))) { - // check that account_id is collab on project_id - throw Error(`user must be collaborator on project with id ${project_id}`); - } - const url = toURL({ project_id, path }); - const data2: embeddings.Data[] = []; - for (const { id, text, meta, hash } of data) { - if (!id || typeof id != "string") { - throw Error( - "you must specify the id for each item and it must be a nonempty string", - ); - } - if (needsText) { - if (!text || typeof text != "string") { - throw Error("each item must have an nonempty text string"); - } - } - data2.push({ - field: "text", - payload: { - text, - url: `${url}#${id}`, - hash, - meta, - }, - }); - } - return data2; -} - -function toURL({ project_id, path }) { - return `\\projects/${project_id}/files/${path}`; -} - -export async function save({ - account_id, - project_id, - path, - data, -}: { - account_id: string; - project_id: string; - path: string; - data: EmbeddingData[]; -}): Promise { - if (data.length == 0) { - // easy - return []; - } - if (data.length > MAX_SAVE_LIMIT) { - throw Error(`you can save at most ${MAX_SAVE_LIMIT} datum in one call`); - } - - const data2: embeddings.Data[] = await prepareData( - account_id, - project_id, - path, - data, - true, - ); - return await embeddings.save(data2, account_id); -} - -// Permanently delete points from vector store that match -export async function remove({ - account_id, - project_id, - path, - data, -}: { - account_id: string; - project_id: string; - path: string; - data: EmbeddingData[]; -}): Promise { - if (data.length == 0) { - // easy - return []; - } - if (data.length > MAX_REMOVE_LIMIT) { - throw Error(`you can remove at most ${MAX_REMOVE_LIMIT} datum in one call`); - } - - const data2 = await prepareData(account_id, project_id, path, data, false); - return await embeddings.remove(data2); -} diff --git a/src/packages/server/llm/embeddings.ts b/src/packages/server/llm/embeddings.ts deleted file mode 100644 index 05ee5251dc..0000000000 --- a/src/packages/server/llm/embeddings.ts +++ /dev/null @@ -1,243 +0,0 @@ -import { sha1, uuidsha1 } from "@cocalc/backend/sha1"; -import { getClient as getDB } from "@cocalc/database/pool"; -import * as qdrant from "@cocalc/database/qdrant"; -import { getClient } from "./client"; -import checkForAbuse from "./embeddings-abuse"; -import { GoogleGenAIClient } from "./google-genai-client"; - -// the vectors we compute using openai's embeddings api get cached for this long -// in our database since they were last accessed. Also, this is how long we -// cache our log of calls. -const EXPIRE = "NOW() + interval '6 weeks'"; - -export interface Data { - payload: qdrant.Payload; - field: string; // payload[field] is the text we encode as a vector -} - -export async function remove(data: Data[]): Promise { - const points = data.map(({ payload }) => getPointId(payload?.url as string)); - await qdrant.deletePoints({ points }); - return points; -} - -export async function save( - data: Data[], - account_id: string, // who is requesting this, so can log it in case we call openai -): Promise { - // Define the Qdrant points that we will be inserting corresponding - // to the given data. - const points: Partial[] = []; - const point_ids: string[] = []; - for (const { payload } of data) { - const point_id = getPointId(payload?.url as string); - point_ids.push(point_id); - points.push({ id: point_id, payload }); - } - - // Now we need the vector component of each of these points. - // These might be available in our cache already, or we - // might have to compute them by calling openai. - const input_sha1s: string[] = []; - const sha1_to_input: { [sha1: string]: string } = {}; - const index_to_sha1: { [n: number]: string } = {}; - let i = 0; - for (const { field, payload } of data) { - if (payload == null) { - throw Error("all payloads must be defined"); - } - const input = payload[field]; - if (typeof input != "string") { - throw Error("payload[field] must be a string"); - } - const s = sha1(input); - input_sha1s.push(s); - sha1_to_input[s] = input; - index_to_sha1[i] = s; - i += 1; - } - // Query database for cached embedding vectors. - const db = getDB(); - try { - await db.connect(); - const { rows } = await db.query( - "SELECT input_sha1,vector FROM openai_embedding_cache WHERE input_sha1 = ANY ($1)", - [input_sha1s], - ); - const sha1_to_vector: { [sha1: string]: number[] } = {}; - for (const { input_sha1, vector } of rows) { - sha1_to_vector[input_sha1] = vector; - } - await db.query( - `UPDATE openai_embedding_cache SET expire=${EXPIRE} WHERE input_sha1 = ANY ($1)`, - [rows.map(({ input_sha1 }) => input_sha1)], - ); - - if (rows.length < data.length) { - // compute some embeddings - const unknown_sha1s = input_sha1s.filter( - (x) => sha1_to_vector[x] == null, - ); - const inputs = unknown_sha1s.map((x) => sha1_to_input[x]); - const vectors = await createEmbeddings(db, inputs, account_id); - for (let i = 0; i < unknown_sha1s.length; i++) { - sha1_to_vector[unknown_sha1s[i]] = vectors[i]; - } - // save the vectors in postgres - await saveEmbeddingsInPostgres(db, unknown_sha1s, vectors); - } - - // Now sha1_to_vector has *all* the vectors in it. - points.map((point, i) => { - point.vector = sha1_to_vector[index_to_sha1[i]]; - }); - - await qdrant.upsert(points as qdrant.Point[]); - return points.map(({ id }) => id as string); - } finally { - db.end(); - } -} - -// a url, but with no special encoding. -// It must always start with a backslash, which is an affordance -// so we can use qdrant to do prefix substring matching. -export function getPointId(url: string) { - if (!url || url[0] != "\\" || url.length <= 1) { - throw Error("url must start with a backslash and be nontrivial"); - } - return uuidsha1(url); -} - -export interface Result { - id: string | number; - payload?: qdrant.Payload; - score?: number; // included for vector search, but NOT for filter search. -} - -// - If id is given search for points near the point with that id. -// - If text is given search for points near the embedding of that search text string -// - If neither id or text is given, then the filter must be given, and find -// points whose payload matches that filter. -// - selector: determines which fields in payload to include/exclude -// - offset: for id/text an integer offset and it reads starting there, just like with SQL; -// for a filter only search, reads points *AFTER* this id (this is a slight change from the -// qdrant api to avoid redundant data transfer to client!!). -export async function search({ - id, - text, - filter, - limit, - selector, - offset, - account_id, -}: { - id?: string; // uuid of a point - text?: string; - filter?: object; - limit: number; - selector?: { include?: string[]; exclude?: string[] }; - offset?: number | string; - account_id: string; // who is doing the search, so we can log this -}): Promise { - if (text != null || id != null) { - let point_id; - if (id != null) { - point_id = id; - } else { - const url = `\\search/${text}`; - // search for points close to text - [point_id] = await save( - [ - { - // time is just to know when this term was last searched, so we could delete stale data if want - payload: { text, time: Date.now(), url }, - field: "text", - }, - ], - account_id, - ); - } - if (typeof offset == "string") { - throw Error( - "when doing a search by text or id, offset must be a number (or not given)", - ); - } - return await qdrant.search({ - id: point_id, - filter, - limit, - selector, - offset, - }); - } else if (filter != null) { - // search using the filter *only*. - // The output of scroll has another property next_page_offset, which - // would be nice to return somehow, which is of course why it is a different - // endpoint for qdrant. Instead, we slightly change how offset works, - // and discard one result. At least the waste stays on the server side. - const { points } = await qdrant.scroll({ - filter, - limit: offset ? limit + 1 : limit, - selector, - offset, - }); - return offset ? points.slice(1) : points; - } else { - throw Error("at least one of id, text or filter MUST be specified"); - } -} - -// get embeddings corresponding to strings. This is just a simple wrapper -// around calling openai, and does not cache anything. -async function createEmbeddings( - db, - input: string[], - account_id: string, -): Promise { - await checkForAbuse(account_id); - // compute embeddings of everythig - const openai = await getClient(); - if (openai instanceof GoogleGenAIClient) { - throw Error("VertexAI not supported"); - } - const response = await openai.embeddings.create({ - model: "text-embedding-ada-002", - input, - }); - - const vectors = response.data.map((x) => x.embedding); - - // log this - await db.query( - `INSERT INTO openai_embedding_log (time,account_id,tokens) VALUES(NOW(),$1,$2)`, - [account_id, response.usage.total_tokens], - ); - return vectors; -} - -async function saveEmbeddingsInPostgres( - db, - input_sha1s: string[], - vectors: number[][], -) { - if (input_sha1s.length == 0) return; - // We don't have to worry about sql injection because all the inputs - // are sha1 hashes and uuid's that we computed. - // Construct the values string for the query. - const sha1s = new Set([]); - const values: string[] = []; - input_sha1s.forEach((input_sha1, i) => { - if (sha1s.has(input_sha1)) return; - sha1s.add(input_sha1); - values.push(`('${input_sha1}', '{${vectors[i].join(",")}}', ${EXPIRE})`); - }); - - // Insert data into the openai_embedding_cache table using a single query - const query = ` - INSERT INTO openai_embedding_cache (input_sha1, vector, expire) - VALUES ${values.join(", ")}; - `; - - await db.query(query); -} diff --git a/src/packages/util/db-schema/site-defaults.ts b/src/packages/util/db-schema/site-defaults.ts index 4536d0d6ae..5d33836813 100644 --- a/src/packages/util/db-schema/site-defaults.ts +++ b/src/packages/util/db-schema/site-defaults.ts @@ -861,7 +861,7 @@ export const site_settings_conf: SiteSettings = { tags: ["AI LLM"], }, neural_search_enabled: { - name: "OpenAI Neural Search UI", + name: "DEPRECATED - OpenAI Neural Search UI", desc: "Controls visibility of UI elements related to Neural Search integration. You must **also set your OpenAI API key** below and fully configure the **Qdrant vector database** for neural search to work.", default: "no", valid: only_booleans, diff --git a/src/packages/util/db-schema/site-settings-extras.ts b/src/packages/util/db-schema/site-settings-extras.ts index 41fe594c0d..541a16a132 100644 --- a/src/packages/util/db-schema/site-settings-extras.ts +++ b/src/packages/util/db-schema/site-settings-extras.ts @@ -337,20 +337,20 @@ export const EXTRAS: SettingsExtras = { tags: ["AI LLM"], }, qdrant_section: { - name: "Qdrant Configuration", + name: "DEPRECATED - Qdrant Configuration", desc: "", default: "", show: neural_search_enabled, type: "header", }, qdrant_cluster_url: { - name: "Qdrant Cluster URL (needed for OpenAI Neural Search)", + name: "DEPRECATED - Qdrant Cluster URL (needed for OpenAI Neural Search)", desc: "Your [Qdrant](https://qdrant.tech/) server from https://cloud.qdrant.io/ or you can also run Qdrant locally. This is needed to support functionality that uses Neural Search.", default: "", show: neural_search_enabled, }, qdrant_api_key: { - name: "Qdrant API key (needed for OpenAI Neural Search)", + name: "DEPRECATED - Qdrant API key (needed for OpenAI Neural Search)", desc: "Your [Qdrant](https://qdrant.tech/) API key, which is needed to connect to your Qdrant server. See https://qdrant.tech/documentation/cloud/cloud-quick-start/#authentication", default: "", password: true, From 0e90dc11eb589adc47ba4d17ee08a3f3acacd7a1 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sat, 12 Oct 2024 21:33:33 +0000 Subject: [PATCH 02/16] chat: fix cursor jumping bug in chat --- src/packages/frontend/chat/input.tsx | 4 +++- src/packages/frontend/chat/message.tsx | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/packages/frontend/chat/input.tsx b/src/packages/frontend/chat/input.tsx index a1a18a91ee..0a0e669f80 100644 --- a/src/packages/frontend/chat/input.tsx +++ b/src/packages/frontend/chat/input.tsx @@ -36,6 +36,7 @@ interface Props { editBarStyle?: CSS; placeholder?: string; autoFocus?: boolean; + moveCursorToEndOfLine?: boolean; } export default function ChatInput({ @@ -55,6 +56,7 @@ export default function ChatInput({ style, submitMentionsRef, syncdb, + moveCursorToEndOfLine, }: Props) { const intl = useIntl(); const onSendRef = useRef(on_send); @@ -82,7 +84,7 @@ export default function ChatInput({ // See https://github.com/sagemathinc/cocalc/issues/6415 const input = dbInput ?? propsInput; setInput(input); - if (input?.trim()) { + if (input?.trim() && moveCursorToEndOfLine) { // have to wait until it's all rendered -- i hate code like this... for (const n of [1, 10, 50]) { setTimeout(() => { diff --git a/src/packages/frontend/chat/message.tsx b/src/packages/frontend/chat/message.tsx index 08b7290f07..c63fd7a83f 100644 --- a/src/packages/frontend/chat/message.tsx +++ b/src/packages/frontend/chat/message.tsx @@ -769,6 +769,7 @@ export default function Message(props: Readonly) { } const replyDate = -getThreadRootDate({ date, messages }); let input; + let moveCursorToEndOfLine = false; if (isLLMThread) { input = ""; } else { @@ -777,6 +778,7 @@ export default function Message(props: Readonly) { input = ""; } else { input = `@${editor_name} `; + moveCursorToEndOfLine = autoFocusReply; } } return ( @@ -784,6 +786,7 @@ export default function Message(props: Readonly) { ) { !props.allowReply || is_folded || props.actions == null - ) + ) { return; + } return (
From 9673e175ec2852513e87551f4a3ec41c516c0d6c Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 13 Oct 2024 04:04:49 +0000 Subject: [PATCH 03/16] chat search -- quick proof of concept --- src/packages/frontend/chat/filter-messages.ts | 26 +- .../frame-editors/chat-editor/editor.ts | 4 + .../frame-editors/chat-editor/search.tsx | 98 +++++++ .../chat-editor/use-search-index.ts | 72 +++++ .../frame-editors/code-editor/actions.ts | 36 +-- .../frame-editors/frame-tree/types.ts | 1 + src/packages/frontend/package.json | 1 + src/packages/pnpm-lock.yaml | 257 +++++------------- 8 files changed, 278 insertions(+), 217 deletions(-) create mode 100644 src/packages/frontend/frame-editors/chat-editor/search.tsx create mode 100644 src/packages/frontend/frame-editors/chat-editor/use-search-index.ts diff --git a/src/packages/frontend/chat/filter-messages.ts b/src/packages/frontend/chat/filter-messages.ts index 8866c86bbd..ef1ac5c530 100644 --- a/src/packages/frontend/chat/filter-messages.ts +++ b/src/packages/frontend/chat/filter-messages.ts @@ -30,7 +30,7 @@ export function filterMessages({ // no filters -- typical special case; waste now time. return messages; } - const searchData = getSearchData(messages); + const searchData = getSearchData({ messages, threads: true }); let matchingRootTimes: Set; if (filter) { matchingRootTimes = new Set(); @@ -107,23 +107,37 @@ type SearchData = { const cache = new LRU({ max: 25 }); -function getSearchData(messages): SearchData { +export function getSearchData({ + messages, + threads, +}: { + messages: ChatMessages; + threads: boolean; +}): SearchData { if (cache.has(messages)) { return cache.get(messages)!; } const data: SearchData = {}; const userMap = redux.getStore("users").get("user_map"); - for (const [time, message] of messages) { + for (let [time, message] of messages) { + if (typeof time != "string") { + // for typescript + time = `${time}`; + } + const messageTime = parseFloat(time); + const content = getContent(message, userMap); + if (!threads) { + data[time] = { content, newestTime: messageTime }; + continue; + } let rootTime: string; if (message.get("reply_to")) { // non-root in thread - rootTime = `${new Date(message.get("reply_to")).valueOf()}`; + rootTime = `${new Date(message.get("reply_to")!).valueOf()}`; } else { // new root thread rootTime = time; } - const messageTime = parseFloat(time); - const content = getContent(message, userMap); if (data[rootTime] == null) { data[rootTime] = { content, diff --git a/src/packages/frontend/frame-editors/chat-editor/editor.ts b/src/packages/frontend/frame-editors/chat-editor/editor.ts index b8d96d138a..f7ca86b114 100644 --- a/src/packages/frontend/frame-editors/chat-editor/editor.ts +++ b/src/packages/frontend/frame-editors/chat-editor/editor.ts @@ -14,6 +14,7 @@ import { createEditor } from "@cocalc/frontend/frame-editors/frame-tree/editor"; import type { EditorDescription } from "@cocalc/frontend/frame-editors/frame-tree/types"; import { terminal } from "@cocalc/frontend/frame-editors/terminal-editor/editor"; import { time_travel } from "@cocalc/frontend/frame-editors/time-travel-editor/editor"; +import { search } from "./search"; const chatroom: EditorDescription = { type: "chatroom", @@ -39,6 +40,7 @@ const chatroom: EditorDescription = { "chatgpt", "scrollToBottom", "scrollToTop", + "show_search", ]), customizeCommands: { scrollToTop: { @@ -59,6 +61,7 @@ const chatroom: EditorDescription = { "increase_font_size", "scrollToTop", "scrollToBottom", + "show_search", ]), } as const; @@ -66,6 +69,7 @@ const EDITOR_SPEC = { chatroom, terminal, time_travel, + search, } as const; export const Editor = createEditor({ diff --git a/src/packages/frontend/frame-editors/chat-editor/search.tsx b/src/packages/frontend/frame-editors/chat-editor/search.tsx new file mode 100644 index 0000000000..75d249898a --- /dev/null +++ b/src/packages/frontend/frame-editors/chat-editor/search.tsx @@ -0,0 +1,98 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +/* +Full text search that is better than a simple filter. +*/ + +import { useFrameContext } from "@cocalc/frontend/frame-editors/frame-tree/frame-context"; +import type { EditorDescription } from "@cocalc/frontend/frame-editors/frame-tree/types"; +import { Button, Card, Input } from "antd"; +import { set } from "@cocalc/util/misc"; +import { useEffect, useMemo, useState } from "react"; +import { throttle } from "lodash"; +import useSearchIndex from "./use-search-index"; +import ShowError from "@cocalc/frontend/components/error"; + +interface Props { + font_size: number; + desc; +} + +function Search({ font_size, desc }: Props) { + const { project_id, path, actions, id } = useFrameContext(); + const [search, setSearch] = useState(desc.get("data-search") ?? ""); + const [result, setResult] = useState(null); + const saveSearch = useMemo( + () => + throttle((search) => { + if (!actions.isClosed()) { + actions.set_frame_data({ id, search }); + } + }, 250), + [project_id, path], + ); + + const { error, setError, index, doRefresh } = useSearchIndex(); + + useEffect(() => { + if (index == null) { + return; + } + if (!search.trim()) { + setResult([]); + return; + } + (async () => { + const result = await index.search({ term: search }); + setResult(result); + })(); + }, [search, index]); + + return ( +
+ + Search {path} + + + } + style={{ fontSize: font_size }} + > + + { + const search = e.target.value ?? ""; + setSearch(search); + saveSearch(search); + }} + /> + + 
+        {JSON.stringify(result, undefined, 2)}
+
+
+ ); +} + +export const search = { + type: "search", + short: "Search", + name: "Search", + icon: "comment", + commands: set(["decrease_font_size", "increase_font_size"]), + component: Search, +} as EditorDescription; diff --git a/src/packages/frontend/frame-editors/chat-editor/use-search-index.ts b/src/packages/frontend/frame-editors/chat-editor/use-search-index.ts new file mode 100644 index 0000000000..7af10daf35 --- /dev/null +++ b/src/packages/frontend/frame-editors/chat-editor/use-search-index.ts @@ -0,0 +1,72 @@ +import { useFrameContext } from "@cocalc/frontend/frame-editors/frame-tree/frame-context"; +import { useEffect, useState } from "react"; +import { create, search, insertMultiple } from "@orama/orama"; +import { getSearchData } from "@cocalc/frontend/chat/filter-messages"; +import useCounter from "@cocalc/frontend/app-framework/counter-hook"; + +export default function useSearchIndex() { + const { actions, project_id, path } = useFrameContext(); + const [index, setIndex] = useState(null); + const [error, setError] = useState(""); + const { val: refresh, inc: doRefresh } = useCounter(); + + useEffect(() => { + (async () => { + try { + setError(""); + const index = new SearchIndex({ actions }); + await index.init(); + setIndex(index); + } catch (err) { + setError(`${err}`); + } + })(); + }, [project_id, path, refresh]); + + return { index, error, doRefresh, setError }; +} + +class SearchIndex { + private actions; + private state: "init" | "ready" | "failed" = "init"; + private error: Error | null = null; + private db; + + constructor({ actions }) { + this.actions = actions; + } + + getState = () => this.state; + getError = () => this.error; + + search = async (query) => { + if (this.state != "ready") { + throw Error("index not ready"); + } + return await search(this.db, query); + }; + + init = async () => { + this.db = await create({ + schema: { + time: "number", + message: "string", + }, + }); + + const messages = this.actions.store?.get("messages"); + if (messages == null) { + return; + } + const searchData = getSearchData({ messages, threads: false }); + const docs: { time: number; message: string }[] = []; + for (const time in searchData) { + docs.push({ + time: parseInt(time), + message: searchData[time]?.content ?? "", + }); + } + await insertMultiple(this.db, docs); + this.state = "ready"; + }; +} diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index 6480691179..b4c6492de4 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -360,7 +360,7 @@ export class Actions< ); return; } - if (!this._syncstring || this._state == "closed") { + if (!this._syncstring || this.isClosed()) { // the doc could perhaps be closed by the time this init is fired, in which case just bail -- no point in trying to initialize anything. return; } @@ -552,8 +552,10 @@ export class Actions< cm.setValue(value); } + isClosed = () => this._state == "closed"; + public close(): void { - if (this._state == "closed") { + if (this.isClosed()) { return; } this._state = "closed"; @@ -1505,7 +1507,7 @@ export class Actions< if (must_create) { // Have to wait until after editor gets created await delay(1); - if (this._state == "closed") return; + if (this.isClosed()) return; } this.programmatical_goto_line(opts.line, opts.cursor); } @@ -1514,7 +1516,7 @@ export class Actions< // Have to wait until after editor gets created, and // probably also event that caused this open. await delay(1); - if (this._state == "closed") return; + if (this.isClosed()) return; const cm = this._recent_cm(); if (cm) { cm.focus(); @@ -1617,7 +1619,7 @@ export class Actions< async set_codemirror_to_syncstring(): Promise { if ( this._syncstring == null || - this._state == "closed" || + this.isClosed() || this._syncstring.get_state() == "closed" ) { // no point in doing anything further. @@ -1626,7 +1628,7 @@ export class Actions< if (this._syncstring.get_state() != "ready") { await once(this._syncstring, "ready"); - if (this._state == "closed") return; + if (this.isClosed()) return; } // NOTE: we fallback to getting the underlying CM doc, in case all actual @@ -1741,7 +1743,7 @@ export class Actions< if (state == "closed") return false; if (state == "init") { await once(syncdoc, "ready"); - if (this._state == "closed") return false; + if (this.isClosed()) return false; } return true; } @@ -1811,7 +1813,7 @@ export class Actions< if (cm == null) { await delay(25); } - if (this._state == "closed") return; + if (this.isClosed()) return; } if (cm == null) { // still failed -- give up. @@ -1833,7 +1835,7 @@ export class Actions< // TODO: this is VERY CRAPPY CODE -- wait after, // so cm gets state/value fully set. await delay(100); - if (this._state == "closed") { + if (this.isClosed()) { return; } doc.setCursor(pos); @@ -1925,7 +1927,7 @@ export class Actions< this.setState({ status }); if (timeout) { await delay(timeout); - if (this._state == "closed") return; + if (this.isClosed()) return; if (this.store.get("status") === status) { this.setState({ status: "" }); } @@ -1977,7 +1979,7 @@ export class Actions< // sets the mispelled_words part of the state to the immutable // Set of those words. They can then be rendered by any editor/view. async update_misspelled_words(time?: number): Promise { - if (this._state == "closed") return; + if (this.isClosed()) return; const proj_store = this.redux.getProjectStore(this.project_id); if (proj_store != null) { // TODO why is this an immutable map? it's project_configuration/Available @@ -2293,7 +2295,7 @@ export class Actions< Exception if can't be done, e.g., if editor not mounted. */ _get_cm_value(): string { - if (this._state == "closed") { + if (this.isClosed()) { throw Error("editor is closed"); } const cm = this._get_cm(); @@ -2311,7 +2313,7 @@ export class Actions< Exception if can't be done. */ _get_syncstring_value(): string { - if (this._state == "closed") { + if (this.isClosed()) { throw Error("editor is closed"); } if (!this._syncstring) { @@ -2459,7 +2461,7 @@ export class Actions< }); await delay(0); // wait until next render loop - if (this._state == "closed") return; + if (this.isClosed()) return; this.set_resize(); this.refresh_visible(); this.focus(); @@ -2562,7 +2564,7 @@ export class Actions< // Have to wait until after editor gets created, and // probably also event that caused this open. await delay(1); - if (this._state == "closed") return; + if (this.isClosed()) return; this.set_active_id(shell_id); } @@ -2584,7 +2586,7 @@ export class Actions< // Have to wait until after editor gets created, and // probably also event that caused this open. await delay(1); - if (this._state == "closed") return; + if (this.isClosed()) return; this.set_active_id(shell_id); } @@ -3076,7 +3078,7 @@ export class Actions< } if (fragmentId.line) { - if (this._state == "closed") return; + if (this.isClosed()) return; this.programmatical_goto_line?.(fragmentId.line, true); } diff --git a/src/packages/frontend/frame-editors/frame-tree/types.ts b/src/packages/frontend/frame-editors/frame-tree/types.ts index 1c120088cb..356349df18 100644 --- a/src/packages/frontend/frame-editors/frame-tree/types.ts +++ b/src/packages/frontend/frame-editors/frame-tree/types.ts @@ -77,6 +77,7 @@ type EditorType = | "rst-view" | "sagews-cells" | "sagews-document" + | "search" | "settings" | "slate" | "slides-notes" diff --git a/src/packages/frontend/package.json b/src/packages/frontend/package.json index 02576bf30a..03f1084c03 100644 --- a/src/packages/frontend/package.json +++ b/src/packages/frontend/package.json @@ -55,6 +55,7 @@ "@jupyter-widgets/output": "^4.1.0", "@lumino/widgets": "^1.31.1", "@microlink/react-json-view": "^1.23.0", + "@orama/orama": "3.0.0-rc-3", "@react-hook/mouse-position": "^4.1.3", "@rinsuki/lz4-ts": "^1.0.1", "@speed-highlight/core": "^1.1.11", diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index df57b1d8c5..279621573d 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -186,7 +186,7 @@ importers: version: 8.7.0 debug: specifier: ^4.3.2 - version: 4.3.7 + version: 4.3.7(supports-color@9.4.0) immutable: specifier: ^4.3.0 version: 4.3.7 @@ -296,6 +296,9 @@ importers: '@microlink/react-json-view': specifier: ^1.23.0 version: 1.23.2(@types/react@18.3.10)(encoding@0.1.13)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@orama/orama': + specifier: 3.0.0-rc-3 + version: 3.0.0-rc-3 '@react-hook/mouse-position': specifier: ^4.1.3 version: 4.1.3(react@18.3.1) @@ -382,7 +385,7 @@ importers: version: 1.11.13 debug: specifier: ^4.3.4 - version: 4.3.7(supports-color@8.1.1) + version: 4.3.7(supports-color@9.4.0) direction: specifier: ^1.0.4 version: 1.0.4 @@ -568,7 +571,7 @@ importers: version: 2.6.0(plotly.js@2.35.2(@rspack/core@1.0.10(@swc/helpers@0.5.13))(mapbox-gl@3.7.0)(webpack@5.95.0))(react@18.3.1) react-redux: specifier: ^8.0.5 - version: 8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@5.0.1) + version: 8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@4.2.1) react-timeago: specifier: ^7.2.0 version: 7.2.0(react@18.3.1) @@ -776,7 +779,7 @@ importers: version: 2.8.5 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@8.1.1) + version: 4.3.7(supports-color@9.4.0) escape-html: specifier: ^1.0.3 version: 1.0.3 @@ -951,7 +954,7 @@ importers: version: 8.7.0 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@8.1.1) + version: 4.3.7(supports-color@9.4.0) enchannel-zmq-backend: specifier: ^9.1.23 version: 9.1.23(rxjs@7.8.1) @@ -1220,7 +1223,7 @@ importers: version: 3.0.0 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@8.1.1) + version: 4.3.7(supports-color@9.4.0) diskusage: specifier: ^1.1.3 version: 1.2.0 @@ -1356,7 +1359,7 @@ importers: version: 0.2.18(encoding@0.1.13)(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)) '@langchain/community': specifier: ^0.2.19 - version: 0.2.33(@google-ai/generativelanguage@2.6.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@qdrant/js-client-rest@1.12.0(typescript@5.6.3))(axios@1.7.7)(better-sqlite3@11.3.0)(cheerio@1.0.0)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.0)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ignore@6.0.2)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.63.0(encoding@0.1.13)(zod@3.23.8))(pg@8.13.0)(ws@8.18.0) + version: 0.2.33(@google-ai/generativelanguage@2.6.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@qdrant/js-client-rest@1.12.0(typescript@5.6.3))(axios@1.7.7)(better-sqlite3@8.7.0)(cheerio@1.0.0-rc.10)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.0)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ignore@6.0.2)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.63.0(encoding@0.1.13)(zod@3.23.8))(pg@8.13.0)(ws@8.18.0) '@langchain/core': specifier: ^0.2.17 version: 0.2.34(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)) @@ -1796,7 +1799,7 @@ importers: version: 3.0.0 debug: specifier: ^4.3.4 - version: 4.3.7(supports-color@8.1.1) + version: 4.3.7(supports-color@9.4.0) events: specifier: 3.3.0 version: 3.3.0 @@ -1848,7 +1851,7 @@ importers: version: 1.0.0 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@8.1.1) + version: 4.3.7(supports-color@9.4.0) primus: specifier: ^8.0.7 version: 8.0.9 @@ -1928,7 +1931,7 @@ importers: version: 3.0.0 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@8.1.1) + version: 4.3.7(supports-color@9.4.0) lodash: specifier: ^4.17.21 version: 4.17.21 @@ -1971,7 +1974,7 @@ importers: version: 1.11.13 debug: specifier: ^4.3.2 - version: 4.3.7(supports-color@8.1.1) + version: 4.3.7(supports-color@9.4.0) events: specifier: 3.3.0 version: 3.3.0 @@ -3494,6 +3497,10 @@ packages: resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} engines: {node: '>=8.0.0'} + '@orama/orama@3.0.0-rc-3': + resolution: {integrity: sha512-09wDVEl32p3Cq84OFoyFzuksP1cSo5AV00mYsBgb0Kqc3qonwdkruQIO1brLDszIXdy3+f2I6AkSKeKhqSRhtA==} + engines: {node: '>= 16.0.0'} + '@parcel/watcher-android-arm64@2.4.1': resolution: {integrity: sha512-LOi/WTbbh3aTn2RYddrO8pnapixAziFl6SMxHM69r3tvdSm94JtCenaKgk1GRg5FJ5wpMCpHeW+7yqPlvZv7kg==} engines: {node: '>= 10.0.0'} @@ -4940,9 +4947,6 @@ packages: bcryptjs@2.4.3: resolution: {integrity: sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==} - better-sqlite3@11.3.0: - resolution: {integrity: sha512-iHt9j8NPYF3oKCNOO5ZI4JwThjt3Z6J6XrcwG85VNMVzv1ByqrHWv5VILEbCMFWDsoHhXvQ7oC8vgRXFAKgl9w==} - better-sqlite3@8.7.0: resolution: {integrity: sha512-99jZU4le+f3G6aIl6PmmV0cxUIWqKieHxsiF7G34CVFiE+/UabpYqkU0NJIkY/96mQKikHeBjtR27vFfs5JpEw==} @@ -5172,10 +5176,6 @@ packages: cheerio-select@2.1.0: resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} - cheerio@1.0.0: - resolution: {integrity: sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==} - engines: {node: '>=18.17'} - cheerio@1.0.0-rc.10: resolution: {integrity: sha512-g0J0q/O6mW8z5zxQ3A8E8J1hUgp4SMOvEoW/x84OwyHKe/Zccz83PVT4y5Crcr530FV6NgmKI1qvGTKVl9XXVw==} engines: {node: '>= 6'} @@ -6244,9 +6244,6 @@ packages: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} - encoding-sniffer@0.2.0: - resolution: {integrity: sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==} - encoding@0.1.13: resolution: {integrity: sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==} @@ -7220,9 +7217,6 @@ packages: htmlparser2@8.0.2: resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} - htmlparser2@9.1.0: - resolution: {integrity: sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==} - htmlparser@1.7.7: resolution: {integrity: sha512-zpK66ifkT0fauyFh2Mulrq4AqGTucxGtOhZ8OjkbSfcCpkqQEI8qRkY0tSQSJNAQ4HUZkgWaU4fK4EH6SVH9PQ==} engines: {node: '>=0.1.33'} @@ -9327,21 +9321,12 @@ packages: parse5-htmlparser2-tree-adapter@7.0.0: resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==} - parse5-htmlparser2-tree-adapter@7.1.0: - resolution: {integrity: sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==} - - parse5-parser-stream@7.1.2: - resolution: {integrity: sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==} - parse5@6.0.1: resolution: {integrity: sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw==} parse5@7.1.2: resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} - parse5@7.2.0: - resolution: {integrity: sha512-ZkDsAOcxsUMZ4Lz5fVciOehNcJ+Gb8gTzcA4yl3wnc273BAybYWrQ+Ks/OjCjSEpjvQkDSeZbybK9qj2VHHdGA==} - parseurl@1.3.3: resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} engines: {node: '>= 0.8'} @@ -10345,9 +10330,6 @@ packages: redux@4.2.1: resolution: {integrity: sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==} - redux@5.0.1: - resolution: {integrity: sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==} - reflect-metadata@0.1.13: resolution: {integrity: sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==} @@ -11472,10 +11454,6 @@ packages: resolution: {integrity: sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==} engines: {node: '>=14.0'} - undici@6.20.0: - resolution: {integrity: sha512-AITZfPuxubm31Sx0vr8bteSalEbs9wQb/BOBi9FPlD9Qpd6HxZ4Q0+hI742jBhkPb4RT2v5MQzaW5VhRVyj+9A==} - engines: {node: '>=18.17'} - unified@10.1.2: resolution: {integrity: sha512-pUSWAi/RAnVy1Pif2kAoeWNBa3JVrx0MId2LASj8G+7AiHWoKZNTomq6LG326T68U7/e263X6fTdcXIy7XnF7Q==} @@ -11814,14 +11792,6 @@ packages: webworkify@1.5.0: resolution: {integrity: sha512-AMcUeyXAhbACL8S2hqqdqOLqvJ8ylmIbNwUIqQujRSouf4+eUFaXbG6F1Rbu+srlJMmxQWsiU7mOJi0nMBfM1g==} - whatwg-encoding@3.1.1: - resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==} - engines: {node: '>=18'} - - whatwg-mimetype@4.0.0: - resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==} - engines: {node: '>=18'} - whatwg-url@5.0.0: resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} @@ -12192,10 +12162,10 @@ snapshots: '@babel/helpers': 7.25.6 '@babel/parser': 7.25.6 '@babel/template': 7.25.0 - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.25.6(supports-color@9.4.0) '@babel/types': 7.25.6 convert-source-map: 2.0.0 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -12235,7 +12205,7 @@ snapshots: '@babel/traverse': 7.25.7 '@babel/types': 7.25.8 convert-source-map: 2.0.0 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -12284,21 +12254,14 @@ snapshots: '@babel/helper-optimise-call-expression': 7.24.7 '@babel/helper-replace-supers': 7.25.0(@babel/core@7.25.2) '@babel/helper-skip-transparent-expression-wrappers': 7.24.7 - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.25.6(supports-color@9.4.0) semver: 6.3.1 transitivePeerDependencies: - supports-color '@babel/helper-member-expression-to-functions@7.24.8': dependencies: - '@babel/traverse': 7.25.6 - '@babel/types': 7.25.6 - transitivePeerDependencies: - - supports-color - - '@babel/helper-module-imports@7.24.7': - dependencies: - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.25.6(supports-color@9.4.0) '@babel/types': 7.25.6 transitivePeerDependencies: - supports-color @@ -12330,10 +12293,10 @@ snapshots: '@babel/helper-module-transforms@7.25.2(@babel/core@7.25.2)': dependencies: '@babel/core': 7.25.2 - '@babel/helper-module-imports': 7.24.7 - '@babel/helper-simple-access': 7.24.7 + '@babel/helper-module-imports': 7.24.7(supports-color@9.4.0) + '@babel/helper-simple-access': 7.24.7(supports-color@9.4.0) '@babel/helper-validator-identifier': 7.24.7 - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.25.6(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -12358,14 +12321,7 @@ snapshots: '@babel/core': 7.25.2 '@babel/helper-member-expression-to-functions': 7.24.8 '@babel/helper-optimise-call-expression': 7.24.7 - '@babel/traverse': 7.25.6 - transitivePeerDependencies: - - supports-color - - '@babel/helper-simple-access@7.24.7': - dependencies: - '@babel/traverse': 7.25.6 - '@babel/types': 7.25.6 + '@babel/traverse': 7.25.6(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -12385,7 +12341,7 @@ snapshots: '@babel/helper-skip-transparent-expression-wrappers@7.24.7': dependencies: - '@babel/traverse': 7.25.6 + '@babel/traverse': 7.25.6(supports-color@9.4.0) '@babel/types': 7.25.6 transitivePeerDependencies: - supports-color @@ -12519,7 +12475,7 @@ snapshots: '@babel/core': 7.25.2 '@babel/helper-module-transforms': 7.25.2(@babel/core@7.25.2) '@babel/helper-plugin-utils': 7.24.8 - '@babel/helper-simple-access': 7.24.7 + '@babel/helper-simple-access': 7.24.7(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -12569,18 +12525,6 @@ snapshots: '@babel/parser': 7.25.8 '@babel/types': 7.25.8 - '@babel/traverse@7.25.6': - dependencies: - '@babel/code-frame': 7.24.7 - '@babel/generator': 7.25.6 - '@babel/parser': 7.25.6 - '@babel/template': 7.25.0 - '@babel/types': 7.25.6 - debug: 4.3.7(supports-color@8.1.1) - globals: 11.12.0 - transitivePeerDependencies: - - supports-color - '@babel/traverse@7.25.6(supports-color@9.4.0)': dependencies: '@babel/code-frame': 7.24.7 @@ -12600,7 +12544,7 @@ snapshots: '@babel/parser': 7.25.8 '@babel/template': 7.25.7 '@babel/types': 7.25.8 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -12630,7 +12574,7 @@ snapshots: awaiting: 3.0.0 cheerio: 1.0.0-rc.12 csv-parse: 5.5.6 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -12644,7 +12588,7 @@ snapshots: '@cocalc/primus-responder@1.0.5': dependencies: - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) node-uuid: 1.4.8 transitivePeerDependencies: - supports-color @@ -12701,7 +12645,7 @@ snapshots: '@eslint/eslintrc@2.1.4': dependencies: ajv: 6.12.6 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) espree: 9.6.1 globals: 13.24.0 ignore: 5.3.1 @@ -12890,7 +12834,7 @@ snapshots: '@humanwhocodes/config-array@0.13.0': dependencies: '@humanwhocodes/object-schema': 2.0.3 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -13021,7 +12965,7 @@ snapshots: istanbul-lib-coverage: 3.2.2 istanbul-lib-instrument: 6.0.3 istanbul-lib-report: 3.0.1 - istanbul-lib-source-maps: 4.0.1 + istanbul-lib-source-maps: 4.0.1(supports-color@9.4.0) istanbul-reports: 3.1.7 jest-message-util: 29.7.0 jest-util: 29.7.0 @@ -13284,7 +13228,7 @@ snapshots: - encoding - openai - '@langchain/community@0.2.33(@google-ai/generativelanguage@2.6.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@qdrant/js-client-rest@1.12.0(typescript@5.6.3))(axios@1.7.7)(better-sqlite3@11.3.0)(cheerio@1.0.0)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.0)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ignore@6.0.2)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.63.0(encoding@0.1.13)(zod@3.23.8))(pg@8.13.0)(ws@8.18.0)': + '@langchain/community@0.2.33(@google-ai/generativelanguage@2.6.0(encoding@0.1.13))(@google-cloud/storage@7.13.0(encoding@0.1.13))(@qdrant/js-client-rest@1.12.0(typescript@5.6.3))(axios@1.7.7)(better-sqlite3@8.7.0)(cheerio@1.0.0-rc.10)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.0)(google-auth-library@9.14.1(encoding@0.1.13))(googleapis@137.1.0(encoding@0.1.13))(handlebars@4.7.8)(ignore@6.0.2)(jsonwebtoken@9.0.2)(lodash@4.17.21)(openai@4.63.0(encoding@0.1.13)(zod@3.23.8))(pg@8.13.0)(ws@8.18.0)': dependencies: '@langchain/core': 0.2.34(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)) '@langchain/openai': 0.2.11(encoding@0.1.13) @@ -13292,7 +13236,7 @@ snapshots: expr-eval: 2.0.2 flat: 5.0.2 js-yaml: 4.1.0 - langchain: 0.2.3(axios@1.7.7)(cheerio@1.0.0)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.0)(handlebars@4.7.8)(ignore@6.0.2)(openai@4.63.0(encoding@0.1.13)(zod@3.23.8))(ws@8.18.0) + langchain: 0.2.3(axios@1.7.7)(cheerio@1.0.0-rc.10)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.0)(handlebars@4.7.8)(ignore@6.0.2)(openai@4.63.0(encoding@0.1.13)(zod@3.23.8))(ws@8.18.0) langsmith: 0.1.59(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)) uuid: 10.0.0 zod: 3.23.8 @@ -13301,8 +13245,8 @@ snapshots: '@google-ai/generativelanguage': 2.6.0(encoding@0.1.13) '@google-cloud/storage': 7.13.0(encoding@0.1.13) '@qdrant/js-client-rest': 1.12.0(typescript@5.6.3) - better-sqlite3: 11.3.0 - cheerio: 1.0.0 + better-sqlite3: 8.7.0 + cheerio: 1.0.0-rc.10 d3-dsv: 3.0.1 google-auth-library: 9.14.1(encoding@0.1.13) googleapis: 137.1.0(encoding@0.1.13) @@ -13629,7 +13573,7 @@ snapshots: '@types/xml-encryption': 1.2.4 '@types/xml2js': 0.4.14 '@xmldom/xmldom': 0.8.10 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) xml-crypto: 3.2.0 xml-encryption: 3.0.2 xml2js: 0.5.0 @@ -13738,6 +13682,8 @@ snapshots: '@opentelemetry/api@1.9.0': optional: true + '@orama/orama@3.0.0-rc-3': {} + '@parcel/watcher-android-arm64@2.4.1': optional: true @@ -14717,7 +14663,7 @@ snapshots: '@typescript-eslint/type-utils': 6.21.0(eslint@8.57.1)(typescript@5.6.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.1)(typescript@5.6.3) '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) eslint: 8.57.1 graphemer: 1.4.0 ignore: 5.3.1 @@ -14735,7 +14681,7 @@ snapshots: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.6.3) '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) eslint: 8.57.1 optionalDependencies: typescript: 5.6.3 @@ -14751,7 +14697,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.6.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.1)(typescript@5.6.3) - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) eslint: 8.57.1 ts-api-utils: 1.3.0(typescript@5.6.3) optionalDependencies: @@ -14765,7 +14711,7 @@ snapshots: dependencies: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.3 @@ -14966,13 +14912,13 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) transitivePeerDependencies: - supports-color agent-base@7.1.1: dependencies: - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -15421,12 +15367,6 @@ snapshots: bcryptjs@2.4.3: {} - better-sqlite3@11.3.0: - dependencies: - bindings: 1.5.0 - prebuild-install: 7.1.2 - optional: true - better-sqlite3@8.7.0: dependencies: bindings: 1.5.0 @@ -15684,21 +15624,6 @@ snapshots: domhandler: 5.0.3 domutils: 3.1.0 - cheerio@1.0.0: - dependencies: - cheerio-select: 2.1.0 - dom-serializer: 2.0.0 - domhandler: 5.0.3 - domutils: 3.1.0 - encoding-sniffer: 0.2.0 - htmlparser2: 9.1.0 - parse5: 7.2.0 - parse5-htmlparser2-tree-adapter: 7.1.0 - parse5-parser-stream: 7.1.2 - undici: 6.20.0 - whatwg-mimetype: 4.0.0 - optional: true - cheerio@1.0.0-rc.10: dependencies: cheerio-select: 1.6.0 @@ -16485,10 +16410,6 @@ snapshots: dependencies: ms: 2.1.2 - debug@4.3.7: - dependencies: - ms: 2.1.3 - debug@4.3.7(supports-color@8.1.1): dependencies: ms: 2.1.3 @@ -16849,12 +16770,6 @@ snapshots: encodeurl@2.0.0: {} - encoding-sniffer@0.2.0: - dependencies: - iconv-lite: 0.6.3 - whatwg-encoding: 3.1.1 - optional: true - encoding@0.1.13: dependencies: iconv-lite: 0.6.3 @@ -17134,7 +17049,7 @@ snapshots: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.2 @@ -17490,7 +17405,7 @@ snapshots: follow-redirects@1.15.6(debug@4.3.7): optionalDependencies: - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) follow-redirects@1.15.9: {} @@ -18242,14 +18157,6 @@ snapshots: domutils: 3.1.0 entities: 4.5.0 - htmlparser2@9.1.0: - dependencies: - domelementtype: 2.3.0 - domhandler: 5.0.3 - domutils: 3.1.0 - entities: 4.5.0 - optional: true - htmlparser@1.7.7: {} http-deceiver@1.2.7: {} @@ -18275,7 +18182,7 @@ snapshots: dependencies: '@tootallnate/once': 2.0.0 agent-base: 6.0.2 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -18302,21 +18209,21 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.2: dependencies: agent-base: 7.1.1 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.4: dependencies: agent-base: 7.1.1 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) transitivePeerDependencies: - supports-color @@ -18746,14 +18653,6 @@ snapshots: make-dir: 4.0.0 supports-color: 7.2.0 - istanbul-lib-source-maps@4.0.1: - dependencies: - debug: 4.3.7(supports-color@8.1.1) - istanbul-lib-coverage: 3.2.2 - source-map: 0.6.1 - transitivePeerDependencies: - - supports-color - istanbul-lib-source-maps@4.0.1(supports-color@9.4.0): dependencies: debug: 4.3.7(supports-color@9.4.0) @@ -19351,7 +19250,7 @@ snapshots: lambda-cloud-node-api@1.0.1: {} - langchain@0.2.3(axios@1.7.7)(cheerio@1.0.0)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.0)(handlebars@4.7.8)(ignore@6.0.2)(openai@4.63.0(encoding@0.1.13)(zod@3.23.8))(ws@8.18.0): + langchain@0.2.3(axios@1.7.7)(cheerio@1.0.0-rc.10)(d3-dsv@3.0.1)(encoding@0.1.13)(fast-xml-parser@4.5.0)(handlebars@4.7.8)(ignore@6.0.2)(openai@4.63.0(encoding@0.1.13)(zod@3.23.8))(ws@8.18.0): dependencies: '@langchain/core': 0.2.34(openai@4.63.0(encoding@0.1.13)(zod@3.23.8)) '@langchain/openai': 0.0.34(encoding@0.1.13) @@ -19371,7 +19270,7 @@ snapshots: zod-to-json-schema: 3.23.0(zod@3.23.8) optionalDependencies: axios: 1.7.7 - cheerio: 1.0.0 + cheerio: 1.0.0-rc.10 d3-dsv: 3.0.1 fast-xml-parser: 4.5.0 handlebars: 4.7.8 @@ -19933,7 +19832,7 @@ snapshots: micromark@3.2.0: dependencies: '@types/debug': 4.1.12 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) decode-named-character-reference: 1.0.2 micromark-core-commonmark: 1.1.0 micromark-factory-space: 1.1.0 @@ -20407,7 +20306,7 @@ snapshots: istanbul-lib-instrument: 4.0.3 istanbul-lib-processinfo: 2.0.3 istanbul-lib-report: 3.0.1 - istanbul-lib-source-maps: 4.0.1 + istanbul-lib-source-maps: 4.0.1(supports-color@9.4.0) istanbul-reports: 3.1.7 make-dir: 3.1.0 node-preload: 0.2.1 @@ -20698,28 +20597,12 @@ snapshots: domhandler: 5.0.3 parse5: 7.1.2 - parse5-htmlparser2-tree-adapter@7.1.0: - dependencies: - domhandler: 5.0.3 - parse5: 7.2.0 - optional: true - - parse5-parser-stream@7.1.2: - dependencies: - parse5: 7.2.0 - optional: true - parse5@6.0.1: {} parse5@7.1.2: dependencies: entities: 4.5.0 - parse5@7.2.0: - dependencies: - entities: 4.5.0 - optional: true - parseurl@1.3.3: {} pascal-case@3.1.2: @@ -21824,7 +21707,7 @@ snapshots: react-property@2.0.0: {} - react-redux@8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@5.0.1): + react-redux@8.1.3(@types/react-dom@18.3.0)(@types/react@18.3.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(redux@4.2.1): dependencies: '@babel/runtime': 7.25.6 '@types/hoist-non-react-statics': 3.3.1 @@ -21837,7 +21720,7 @@ snapshots: '@types/react': 18.3.10 '@types/react-dom': 18.3.0 react-dom: 18.3.1(react@18.3.1) - redux: 5.0.1 + redux: 4.2.1 react-refresh@0.14.2: {} @@ -21950,9 +21833,6 @@ snapshots: dependencies: '@babel/runtime': 7.25.6 - redux@5.0.1: - optional: true - reflect-metadata@0.1.13: {} reflect.getprototypeof@1.0.6: @@ -22560,7 +22440,7 @@ snapshots: spdy-transport@3.0.0: dependencies: - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) detect-node: 2.1.0 hpack.js: 2.1.6 obuf: 1.1.2 @@ -22571,7 +22451,7 @@ snapshots: spdy@4.0.2: dependencies: - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) handle-thing: 2.0.1 http-deceiver: 1.2.7 select-hose: 2.0.0 @@ -23218,9 +23098,6 @@ snapshots: '@fastify/busboy': 2.1.0 optional: true - undici@6.20.0: - optional: true - unified@10.1.2: dependencies: '@types/unist': 2.0.11 @@ -23688,7 +23565,7 @@ snapshots: dependencies: '@wwa/statvfs': 1.1.18 awaiting: 3.0.0 - debug: 4.3.7(supports-color@8.1.1) + debug: 4.3.7(supports-color@9.4.0) port-get: 1.0.4 ws: 8.18.0 transitivePeerDependencies: @@ -23698,14 +23575,6 @@ snapshots: webworkify@1.5.0: {} - whatwg-encoding@3.1.1: - dependencies: - iconv-lite: 0.6.3 - optional: true - - whatwg-mimetype@4.0.0: - optional: true - whatwg-url@5.0.0: dependencies: tr46: 0.0.3 From 46b1124c7df4c87fcc2eb8bac0ad8713c4807211 Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 13 Oct 2024 14:54:08 +0000 Subject: [PATCH 04/16] chat: fix Message component to use standard style --- src/packages/frontend/chat/chat-log.tsx | 5 - src/packages/frontend/chat/message.tsx | 208 ++++++++---------- .../frame-editors/chat-editor/search.tsx | 35 ++- .../chat-editor/use-search-index.ts | 5 +- 4 files changed, 122 insertions(+), 131 deletions(-) diff --git a/src/packages/frontend/chat/chat-log.tsx b/src/packages/frontend/chat/chat-log.tsx index 7820a74a6a..d3b5c3133c 100644 --- a/src/packages/frontend/chat/chat-log.tsx +++ b/src/packages/frontend/chat/chat-log.tsx @@ -544,11 +544,6 @@ export function MessageList({ sortedDates, messages, )} - is_next_sender={isNextMessageSender( - index, - sortedDates, - messages, - )} show_avatar={!isNextMessageSender(index, sortedDates, messages)} mode={mode} get_user_name={(account_id: string | undefined) => diff --git a/src/packages/frontend/chat/message.tsx b/src/packages/frontend/chat/message.tsx index c63fd7a83f..e83073333b 100644 --- a/src/packages/frontend/chat/message.tsx +++ b/src/packages/frontend/chat/message.tsx @@ -108,7 +108,6 @@ interface Props { path?: string; font_size: number; is_prev_sender?: boolean; - is_next_sender?: boolean; show_avatar?: boolean; mode: Mode; selectedHashtags?: Set; @@ -128,20 +127,29 @@ interface Props { selected?: boolean; } -export default function Message(props: Readonly) { - const { - is_folded, - is_thread_body, - is_thread, - message, - messages, - mode, - project_id, - font_size, - selected, - costEstimate, - } = props; - +export default function Message({ + index, + actions, + get_user_name, + messages, + message, + account_id, + user_map, + project_id, + path, + font_size, + is_prev_sender, + show_avatar, + mode, + selectedHashtags, + scroll_into_view, + allowReply, + is_thread, + is_folded, + is_thread_body, + costEstimate, + selected, +}: Props) { const showAISummarize = redux .getStore("projects") .hasLanguageModelEnabled(project_id, "chat-summarize"); @@ -174,29 +182,27 @@ export default function Message(props: Readonly) { const history_size = useMemo(() => message.get("history").size, [message]); const isEditing = useMemo( - () => is_editing(message, props.account_id), - [message, props.account_id], + () => is_editing(message, account_id), + [message, account_id], ); const editor_name = useMemo(() => { - return props.get_user_name( - message.get("history")?.first()?.get("author_id"), - ); + return get_user_name(message.get("history")?.first()?.get("author_id")); }, [message]); const reverseRowOrdering = - !is_thread_body && sender_is_viewer(props.account_id, message); + !is_thread_body && sender_is_viewer(account_id, message); const submitMentionsRef = useRef(); const [replying, setReplying] = useState(() => { - if (!props.allowReply) { + if (!allowReply) { return false; } const replyDate = -getThreadRootDate({ date, messages }); - const draft = props.actions?.syncdb?.get_one({ + const draft = actions?.syncdb?.get_one({ event: "draft", - sender_id: props.account_id, + sender_id: account_id, date: replyDate, }); if (draft == null) { @@ -210,10 +216,10 @@ export default function Message(props: Readonly) { return true; }); useEffect(() => { - if (!props.allowReply) { + if (!allowReply) { setReplying(false); } - }, [props.allowReply]); + }, [allowReply]); const [autoFocusReply, setAutoFocusReply] = useState(false); const [autoFocusEdit, setAutoFocusEdit] = useState(false); @@ -221,12 +227,12 @@ export default function Message(props: Readonly) { const replyMessageRef = useRef(""); const replyMentionsRef = useRef(); - const is_viewers_message = sender_is_viewer(props.account_id, message); + const is_viewers_message = sender_is_viewer(account_id, message); const verb = show_history ? "Hide" : "Show"; const isLLMThread = useMemo( - () => props.actions?.isLanguageModelThread(message.get("date")), - [message, props.actions != null], + () => actions?.isLanguageModelThread(message.get("date")), + [message, actions != null], ); const msgWrittenByLLM = useMemo(() => { @@ -236,7 +242,7 @@ export default function Message(props: Readonly) { useLayoutEffect(() => { if (replying) { - props.scroll_into_view?.(); + scroll_into_view?.(); } }, [replying]); @@ -244,7 +250,7 @@ export default function Message(props: Readonly) { let text; const other_editors = message .get("editing") - .remove(props.account_id) + .remove(account_id) // @ts-ignore – not sure why this error shows up .keySeq(); if (is_editing) { @@ -252,7 +258,7 @@ export default function Message(props: Readonly) { // This user and someone else is also editing text = ( <> - {`WARNING: ${props.get_user_name( + {`WARNING: ${get_user_name( other_editors.first(), )} is also editing this! `} Simultaneous editing of messages is not supported. @@ -273,7 +279,7 @@ export default function Message(props: Readonly) { } else { if (other_editors.size === 1) { // One person is editing - text = `${props.get_user_name( + text = `${get_user_name( other_editors.first(), )} is editing this message`; } else if (other_editors.size > 1) { @@ -330,30 +336,26 @@ export default function Message(props: Readonly) { } function edit_message() { - if ( - props.project_id == null || - props.path == null || - props.actions == null - ) { + if (project_id == null || path == null || actions == null) { // no editing functionality or not in a project with a path. return; } - props.actions.setEditing(message, true); + actions.setEditing(message, true); setAutoFocusEdit(true); - props.scroll_into_view?.(); + scroll_into_view?.(); } function avatar_column() { const sender_id = message.get("sender_id"); let style: CSSProperties = {}; - if (!props.is_prev_sender) { + if (!is_prev_sender) { style.marginTop = "22px"; } else { style.marginTop = "5px"; } if (!is_thread_body) { - if (sender_is_viewer(props.account_id, message)) { + if (sender_is_viewer(account_id, message)) { style.marginLeft = AVATAR_MARGIN_LEFTRIGHT; } else { style.marginRight = AVATAR_MARGIN_LEFTRIGHT; @@ -363,7 +365,7 @@ export default function Message(props: Readonly) { return (
- {sender_id != null && props.show_avatar ? ( + {sender_id != null && show_avatar ? ( ) : undefined}
@@ -376,19 +378,17 @@ export default function Message(props: Readonly) { let value = newest_content(message); const { background, color, lighten, message_class } = message_colors( - props.account_id, + account_id, message, ); - const font_size = `${props.font_size}px`; - - if (props.show_avatar) { + if (show_avatar) { marginBottom = "1vh"; } else { marginBottom = "3px"; } - if (!props.is_prev_sender && is_viewers_message) { + if (!is_prev_sender && is_viewers_message) { marginTop = MARGIN_TOP_VIEWER; } else { marginTop = "5px"; @@ -401,7 +401,7 @@ export default function Message(props: Readonly) { borderRadius: "5px", marginBottom, marginTop, - fontSize: font_size, + fontSize: `${font_size}px`, padding: selected ? "6px" : "9px", ...(mode === "sidechat" ? { marginLeft: "5px", marginRight: "5px" } @@ -411,7 +411,7 @@ export default function Message(props: Readonly) { const mainXS = mode === "standalone" ? 20 : 22; const showEditButton = Date.now() - date < SHOW_EDIT_BUTTON_MS; - const feedback = message.getIn(["feedback", props.account_id]); + const feedback = message.getIn(["feedback", account_id]); const otherFeedback = isLLMThread && msgWrittenByLLM ? 0 : (message.get("feedback")?.size ?? 0); const showOtherFeedback = otherFeedback > 0; @@ -463,7 +463,7 @@ export default function Message(props: Readonly) { }} type="text" size="small" - onClick={() => props.actions?.setEditing(message, true)} + onClick={() => actions?.setEditing(message, true)} > Edit @@ -478,8 +478,8 @@ export default function Message(props: Readonly) { title="Delete this message" description="Are you sure you want to delete this message?" onConfirm={() => { - props.actions?.setEditing(message, true); - setTimeout(() => props.actions?.sendEdit(message, ""), 1); + actions?.setEditing(message, true); + setTimeout(() => actions?.sendEdit(message, ""), 1); }} > {showAISummarize && is_thread ? ( - + ) : undefined}
); @@ -934,7 +914,7 @@ export default function Message(props: Readonly) { type="text" icon={} onClick={() => - props.actions?.toggleFoldThread(message.get("date"), props.index) + actions?.toggleFoldThread(message.get("date"), index) } > Unfold @@ -968,9 +948,7 @@ export default function Message(props: Readonly) { + + } style={{ fontSize: font_size }} > - + (null); const [error, setError] = useState(""); const { val: refresh, inc: doRefresh } = useCounter(); + const [indexTime, setIndexTime] = useState(0); useEffect(() => { (async () => { try { setError(""); + const t0 = Date.now(); const index = new SearchIndex({ actions }); await index.init(); setIndex(index); + setIndexTime(Date.now() - t0); } catch (err) { setError(`${err}`); } })(); }, [project_id, path, refresh]); - return { index, error, doRefresh, setError }; + return { index, error, doRefresh, setError, indexTime }; } class SearchIndex { From 3326d7cce91a298bd0a63ecb79109874d7939cee Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 13 Oct 2024 15:22:31 +0000 Subject: [PATCH 05/16] chat: rewrite getSortedDates / isThread to be vastly more efficient and scalable - this was easy (with coffee) and an obvious todo. --- src/packages/frontend/chat/chat-log.tsx | 74 +++++++++++++++++-------- src/packages/frontend/chat/message.tsx | 4 +- 2 files changed, 53 insertions(+), 25 deletions(-) diff --git a/src/packages/frontend/chat/chat-log.tsx b/src/packages/frontend/chat/chat-log.tsx index d3b5c3133c..33a42b99ba 100644 --- a/src/packages/frontend/chat/chat-log.tsx +++ b/src/packages/frontend/chat/chat-log.tsx @@ -77,11 +77,16 @@ export function ChatLog({ const user_map = useTypedRedux("users", "user_map"); const account_id = useTypedRedux("account", "account_id"); - const { dates: sortedDates, numFolded } = useMemo<{ + const { + dates: sortedDates, + numFolded, + numChildren, + } = useMemo<{ dates: string[]; numFolded: number; + numChildren; }>(() => { - const { dates, numFolded } = getSortedDates( + const { dates, numFolded, numChildren } = getSortedDates( messages, search, account_id!, @@ -97,7 +102,7 @@ export function ChatLog({ : new Date(parseFloat(dates[dates.length - 1])), ); }, 1); - return { dates, numFolded }; + return { dates, numFolded, numChildren }; }, [messages, search, project_id, path, filterRecentH]); useEffect(() => { @@ -237,6 +242,7 @@ export function ChatLog({ manualScrollRef, mode, selectedDate, + numChildren, }} /> m.get("reply_to") === s); + return (numChildren[message.get("date").valueOf()] ?? 0) > 0; } function isFolded( @@ -323,24 +322,45 @@ export function getSortedDates( search: string | undefined, account_id: string, filterRecentH?: number, -): { dates: string[]; numFolded: number } { +): { + dates: string[]; + numFolded: number; + numChildren: { [date: number]: number }; +} { let numFolded = 0; let m = messages; if (m == null) { - return { dates: [], numFolded: 0 }; + return { + dates: [], + numFolded: 0, + numChildren: {}, + }; } + // we assume filterMessages contains complete threads. It does + // right now, but that's an assumption in this function. m = filterMessages({ messages: m, filter: search, filterRecentH }); + // Do a linear pass through all messages to divide into threads, so that + // getSortedDates is O(n) instead of O(n^2) ! + const numChildren: { [date: number]: number } = {}; + for (const [_, message] of m) { + const parent = message.get("reply_to"); + if (parent != null) { + const d = new Date(parent).valueOf(); + numChildren[d] = (numChildren[d] ?? 0) + 1; + } + } + const v: [date: number, reply_to: number | undefined][] = []; for (const [date, message] of m) { if (message == null) continue; // If we search for a message, we treat all threads as unfolded if (!search) { - const is_thread = isThread(messages, message); - const is_folded = isFolded(messages, message, account_id); - const is_thread_body = message.get("reply_to") != null; + const is_thread = isThread(message, numChildren); + const is_folded = is_thread && isFolded(messages, message, account_id); + const is_thread_body = is_thread && message.get("reply_to") != null; const folded = is_thread && is_folded && is_thread_body; if (folded) { numFolded++; @@ -356,7 +376,7 @@ export function getSortedDates( } v.sort(cmpMessages); const dates = v.map((z) => `${z[0]}`); - return { dates, numFolded }; + return { dates, numFolded, numChildren }; } /* @@ -465,6 +485,7 @@ export function MessageList({ manualScrollRef, mode, selectedDate, + numChildren, }: { messages; account_id; @@ -481,6 +502,7 @@ export function MessageList({ costEstimate?; manualScrollRef?; selectedDate?: string; + numChildren?; }) { const virtuosoHeightsRef = useRef<{ [index: number]: number }>({}); const virtuosoScroll = useVirtuosoScrollHook({ @@ -510,9 +532,13 @@ export function MessageList({ return
; } - const is_thread = isThread(messages, message); - const is_folded = isFolded(messages, message, account_id); - const is_thread_body = message.get("reply_to") != null; + // only do threading if numChildren is defined. It's not defined, + // e.g., when viewing past versions via TimeTravel. + const is_thread = numChildren != null && isThread(message, numChildren); + // optimization: only threads can be folded, so don't waste time + // checking on folding state if it isn't a thread. + const is_folded = is_thread && isFolded(messages, message, account_id); + const is_thread_body = is_thread && message.get("reply_to") != null; const h = virtuosoHeightsRef.current[index]; return ( diff --git a/src/packages/frontend/chat/message.tsx b/src/packages/frontend/chat/message.tsx index e83073333b..b5bd0171a6 100644 --- a/src/packages/frontend/chat/message.tsx +++ b/src/packages/frontend/chat/message.tsx @@ -904,7 +904,9 @@ export default function Message({ } function renderFoldedRow() { - if (!is_folded || !is_thread || is_thread_body) return; + if (!is_folded || !is_thread || is_thread_body) { + return; + } return ( From 0e8a70b0605bc84c6ca853dac07abf5b1604dfcf Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 13 Oct 2024 15:46:59 +0000 Subject: [PATCH 06/16] chat: more useful info about folded thread --- src/packages/frontend/chat/chat-log.tsx | 1 + src/packages/frontend/chat/message.tsx | 54 +++++++++++++------------ 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/packages/frontend/chat/chat-log.tsx b/src/packages/frontend/chat/chat-log.tsx index 33a42b99ba..caa21ce18e 100644 --- a/src/packages/frontend/chat/chat-log.tsx +++ b/src/packages/frontend/chat/chat-log.tsx @@ -551,6 +551,7 @@ export function MessageList({ 's, which have a big 10px margin on their bottoms + // already. + padding: selected ? "6px 6px 0 6px" : "9px 9px 0 9px", ...(mode === "sidechat" ? { marginLeft: "5px", marginRight: "5px" } : undefined), @@ -908,20 +902,30 @@ export default function Message({ return; } + let label; + if (numChildren) { + label = ( + <> + {numChildren} {plural(numChildren, "Reply", "Replies")} + + ); + } else { + label = "View Replies"; + } + return ( - - {mode === "standalone" ? "This thread is folded. " : ""} +
- +
); } @@ -965,7 +969,7 @@ export default function Message({ key={"blankcolumn"} style={{ textAlign: reverseRowOrdering ? "left" : "right" }} > - {true || hideTooltip ? ( + {hideTooltip ? ( button ) : ( Date: Sun, 13 Oct 2024 16:02:09 +0000 Subject: [PATCH 07/16] chat: fix some spacing and tips --- src/packages/frontend/chat/message.tsx | 43 ++++++++++++++++++-------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/src/packages/frontend/chat/message.tsx b/src/packages/frontend/chat/message.tsx index a598e59eba..e86750a84f 100644 --- a/src/packages/frontend/chat/message.tsx +++ b/src/packages/frontend/chat/message.tsx @@ -606,16 +606,20 @@ export default function Message({ ) : ( "" )} - + + + )}{" "} - + - - + <>Search Chatroom {separate_file_extension(path_split(path).tail).name} } style={{ fontSize: font_size }} > @@ -90,7 +81,7 @@ function Search({ font_size, desc }: Props) { { const search = e.target.value ?? ""; @@ -117,7 +108,7 @@ function SearchResult({ hit, actions }) {
= React.memo( useEffect(() => { if (!frameRootRef.current) return; const observer = new ResizeObserver(() => { - actions.set_resize(); + actions.set_resize?.(); }); observer.observe(frameRootRef.current); return () => observer.disconnect(); diff --git a/src/packages/frontend/frame-editors/frame-tree/frame-tree-drag-bar.tsx b/src/packages/frontend/frame-editors/frame-tree/frame-tree-drag-bar.tsx index 521c801c8f..6269ee41cb 100644 --- a/src/packages/frontend/frame-editors/frame-tree/frame-tree-drag-bar.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/frame-tree-drag-bar.tsx @@ -90,7 +90,7 @@ export const FrameTreeDragBar: React.FC = React.memo((props: Props) => { id: frame_tree.get("id"), pos, }); - actions.set_resize(); + actions.set_resize?.(); actions.focus(); // see https://github.com/sagemathinc/cocalc/issues/3269 } diff --git a/src/packages/frontend/frame-editors/frame-tree/register.ts b/src/packages/frontend/frame-editors/frame-tree/register.ts index 9545ff293d..3a73185146 100644 --- a/src/packages/frontend/frame-editors/frame-tree/register.ts +++ b/src/packages/frontend/frame-editors/frame-tree/register.ts @@ -37,7 +37,7 @@ interface Register { } function isAsyncRegister( - opts: Register | AsyncRegister + opts: Register | AsyncRegister, ): opts is AsyncRegister { return opts["editor"] != null; } @@ -71,7 +71,7 @@ export function register_file_editor(opts: Register | AsyncRegister) { opts.component, opts.Actions, opts.asyncData, - is_public + is_public, ); } } @@ -95,7 +95,7 @@ function register( component: any; Actions: any; }>), - is_public: boolean + is_public: boolean, ) { let data: any = { icon, @@ -144,9 +144,7 @@ function register( if (is_public) return; const name = redux_name(project_id, path); const actions = redux.getActions(name); - if (actions) { - actions.save(); - } + actions?.save?.(); }, }; @@ -179,7 +177,7 @@ function register( } else { if (asyncData == null) { throw Error( - "either asyncData must be given or components and Actions must be given (or both)" + "either asyncData must be given or components and Actions must be given (or both)", ); } let async_data: any = undefined; diff --git a/src/scripts/g.sh b/src/scripts/g.sh index 81351fc356..79310bfe67 100755 --- a/src/scripts/g.sh +++ b/src/scripts/g.sh @@ -10,7 +10,6 @@ unset DEBUG_CONSOLE #export COCALC_DISABLE_API_VALIDATION=yes #export NO_RSPACK_DEV_SERVER=yes -#ulimit -Sv 120000000 while true; do pnpm hub From 114b9791cf697d262ff59109ec4dc5a21429063a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 13 Oct 2024 21:21:43 +0000 Subject: [PATCH 12/16] tasks: implement full fragment support... - worth spending on hour on... --- .../frontend/editors/task-editor/actions.ts | 120 ++++++++++++++---- .../frontend/editors/task-editor/editor.tsx | 4 - .../frontend/editors/task-editor/list.tsx | 21 +-- .../frontend/editors/task-editor/types.ts | 4 +- .../frame-editors/task-editor/actions.ts | 40 ++++-- 5 files changed, 130 insertions(+), 59 deletions(-) diff --git a/src/packages/frontend/editors/task-editor/actions.ts b/src/packages/frontend/editors/task-editor/actions.ts index 3a5ce63e6e..f70dc86cfa 100644 --- a/src/packages/frontend/editors/task-editor/actions.ts +++ b/src/packages/frontend/editors/task-editor/actions.ts @@ -7,9 +7,6 @@ Task Actions */ -const LAST_EDITED_THRESH_S = 30; -const TASKS_HELP_URL = "https://doc.cocalc.com/tasks.html"; - import { fromJS, Map } from "immutable"; import { throttle } from "lodash"; import { delay } from "awaiting"; @@ -26,6 +23,7 @@ import { create_key_handler } from "./keyboard"; import { toggle_checkbox } from "./desc-rendering"; import { Actions } from "../../app-framework"; import { + Align, HashtagState, Headings, HeadingsDir, @@ -40,6 +38,10 @@ import { TaskStore } from "./store"; import { SyncDB } from "@cocalc/sync/editor/db"; import { webapp_client } from "../../webapp-client"; import type { Actions as TaskFrameActions } from "@cocalc/frontend/frame-editors/task-editor/actions"; +import Fragment from "@cocalc/frontend/misc/fragment-id"; + +const LAST_EDITED_THRESH_S = 30; +const TASKS_HELP_URL = "https://doc.cocalc.com/tasks.html"; export class TaskActions extends Actions { public syncdb: SyncDB; @@ -53,13 +55,14 @@ export class TaskActions extends Actions { private set_save_status?: () => void; private frameId: string; private frameActions: TaskFrameActions; + private virtuosoRef?; public _init( project_id: string, path: string, syncdb: SyncDB, store: TaskStore, - truePath: string // because above path is auxpath for each frame. + truePath: string, // because above path is auxpath for each frame. ): void { this._update_visible = throttle(this.__update_visible, 500); this.project_id = project_id; @@ -136,7 +139,7 @@ export class TaskActions extends Actions { local_task_state, view, counts, - current_task_id + current_task_id, ); if (obj.visible.size == 0 && view.get("search")?.trim().length == 0) { @@ -148,7 +151,7 @@ export class TaskActions extends Actions { local_task_state, view, counts, - current_task_id + current_task_id, ); } @@ -229,6 +232,21 @@ export class TaskActions extends Actions { } } + clearAllFilters = (obj?) => { + this.set_local_view_state( + { + show_deleted: false, + show_done: false, + show_max: false, + selected_hashtags: {}, + search: "", + ...obj, + }, + false, + ); + this.__update_visible(); + }; + public async save(): Promise { if (this.is_closed) { return; @@ -297,7 +315,7 @@ export class TaskActions extends Actions { task_id?: string, obj?: object, setState: boolean = false, - save: boolean = true // make new commit to syncdb state + save: boolean = true, // make new commit to syncdb state ): void { if (obj == null || this.is_closed) { return; @@ -334,7 +352,7 @@ export class TaskActions extends Actions { // **immediately**; this would happen // eventually as a result of the syncdb set above. let tasks = this.store.get("tasks") ?? fromJS({}); - task = tasks.get(task_id) ?? fromJS({ task_id }) as any; + task = tasks.get(task_id) ?? (fromJS({ task_id }) as any); if (task == null) throw Error("bug"); for (let k in obj) { const v = obj[k]; @@ -405,7 +423,7 @@ export class TaskActions extends Actions { const pos_j = tasks.getIn([visible.get(j), "position"]); this.set_task(task_id, { position: pos_j }, true); this.set_task(visible.get(j), { position: pos_i }, true); - this.scroll_into_view(); + this.scrollIntoView(); } public time_travel(): void { @@ -419,11 +437,14 @@ export class TaskActions extends Actions { window.open(TASKS_HELP_URL, "_blank")?.focus(); } - public set_current_task(task_id: string): void { - if (this.getFrameData("current_task_id") == task_id) return; + set_current_task = (task_id: string): void => { + if (this.getFrameData("current_task_id") == task_id) { + return; + } this.setFrameData({ current_task_id: task_id }); - this.scroll_into_view(); - } + this.scrollIntoView(); + this.setFragment(task_id); + }; public set_current_task_delta(delta: number): void { const task_id = this.getFrameData("current_task_id"); @@ -492,7 +513,7 @@ export class TaskActions extends Actions { this.set_task( task_id, { done: !this.store.getIn(["tasks", task_id, "done"]) }, - true + true, ); } } @@ -523,7 +544,7 @@ export class TaskActions extends Actions { public set_due_date( task_id: string | undefined, - date: number | undefined + date: number | undefined, ): void { this.set_task(task_id, { due_date: date }); } @@ -531,7 +552,7 @@ export class TaskActions extends Actions { public set_desc( task_id: string | undefined, desc: string, - save: boolean = true + save: boolean = true, ): void { this.set_task(task_id, { desc }, false, save); } @@ -646,14 +667,57 @@ export class TaskActions extends Actions { this.setFrameData({ focus_find_box: false }); } - async scroll_into_view(): Promise { - await delay(50); - this.setFrameData({ scroll_into_view: true }); - } + setVirtuosoRef = (virtuosoRef) => { + this.virtuosoRef = virtuosoRef; + }; - public scroll_into_view_done(): void { - this.setFrameData({ scroll_into_view: false }); - } + // scroll the current_task_id into view, possibly changing filters + // in order to make it visibile, if necessary. + scrollIntoView = async (align: Align = "view") => { + if (this.virtuosoRef?.current == null) { + return; + } + const current_task_id = this.getFrameData("current_task_id"); + if (current_task_id == null) { + return; + } + let visible = this.getFrameData("visible"); + if (visible == null) { + return; + } + // Figure out the index of current_task_id. + let index = visible.indexOf(current_task_id); + if (index === -1) { + const task = this.store.getIn(["tasks", current_task_id]); + if (task == null) { + // no such task anywhere, not even in trash, etc + return; + } + if ( + this.getFrameData("search_desc")?.trim() || + task.get("deleted") || + task.get("done") + ) { + // active search -- try clearing it. + this.clearAllFilters({ + show_deleted: !!task.get("deleted"), + show_done: !!task.get("done"), + }); + visible = this.getFrameData("visible"); + index = visible.indexOf(current_task_id); + if (index == -1) { + return; + } + } else { + return; + } + } + if (align == "start" || align == "center" || align == "end") { + this.virtuosoRef.current.scrollToIndex({ index, align }); + } else { + this.virtuosoRef.current.scrollIntoView({ index }); + } + }; public set_show_max(show_max: number): void { this.set_local_view_state({ show_max }, false); @@ -669,7 +733,7 @@ export class TaskActions extends Actions { public toggle_desc_checkbox( task_id: string, index: number, - checked: boolean + checked: boolean, ): void { let desc = this.store.getIn(["tasks", task_id, "desc"]); if (desc == null) { @@ -738,6 +802,14 @@ export class TaskActions extends Actions { .getProjectActions(this.project_id) .open_file({ path, foreground: true }); } + + setFragment = (id?) => { + if (!id) { + Fragment.clear(); + } else { + Fragment.set({ id }); + } + }; } function getPositions(tasks): number[] { diff --git a/src/packages/frontend/editors/task-editor/editor.tsx b/src/packages/frontend/editors/task-editor/editor.tsx index b412e94a50..1a9c7b6a1e 100644 --- a/src/packages/frontend/editors/task-editor/editor.tsx +++ b/src/packages/frontend/editors/task-editor/editor.tsx @@ -34,9 +34,7 @@ interface Props { export const TaskEditor: React.FC = React.memo( ({ actions, path, project_id, desc, read_only }) => { const useEditor = useEditorRedux({ project_id, path }); - const tasks = useEditor("tasks"); - const visible = desc.get("data-visible"); const local_task_state = desc.get("data-local_task_state") ?? fromJS({}); const local_view_state = desc.get("data-local_view_state") ?? fromJS({}); @@ -46,7 +44,6 @@ export const TaskEditor: React.FC = React.memo( const search_terms = desc.get("data-search_terms"); const search_desc = desc.get("data-search_desc"); const focus_find_box = desc.get("data-focus_find_box"); - const scroll_into_view = desc.get("data-scroll_into_view"); if (tasks == null || visible == null) { return ( @@ -126,7 +123,6 @@ export const TaskEditor: React.FC = React.memo( current_task_id={current_task_id} local_task_state={local_task_state} scrollState={(local_view_state as any).get("scrollState")?.toJS?.()} - scroll_into_view={scroll_into_view} font_size={desc.get("font_size")} sortable={ !read_only && diff --git a/src/packages/frontend/editors/task-editor/list.tsx b/src/packages/frontend/editors/task-editor/list.tsx index aa9e8e3dd0..b867cdd51d 100644 --- a/src/packages/frontend/editors/task-editor/list.tsx +++ b/src/packages/frontend/editors/task-editor/list.tsx @@ -31,7 +31,6 @@ interface Props { current_task_id?: string; local_task_state?: LocalTaskStateMap; scrollState?: any; - scroll_into_view?: boolean; font_size: number; sortable?: boolean; read_only?: boolean; @@ -48,7 +47,6 @@ export default function TaskList({ current_task_id, local_task_state, scrollState, - scroll_into_view, font_size, sortable, read_only, @@ -90,23 +88,8 @@ export default function TaskList({ }, [search_terms]); useEffect(() => { - if (actions && scroll_into_view) { - _scroll_into_view(); - actions.scroll_into_view_done(); - } - }, [scroll_into_view]); - - function _scroll_into_view() { - if (current_task_id == null) { - return; - } - // Figure out the index of current_task_id. - const index = visible.indexOf(current_task_id); - if (index === -1) { - return; - } - virtuosoRef.current?.scrollIntoView({ index }); - } + actions?.setVirtuosoRef(virtuosoRef); + }, [actions, virtuosoRef]); function render_task(task_id, index?) { if (index === visible.size) { diff --git a/src/packages/frontend/editors/task-editor/types.ts b/src/packages/frontend/editors/task-editor/types.ts index 9466c9158e..6a2a573db1 100644 --- a/src/packages/frontend/editors/task-editor/types.ts +++ b/src/packages/frontend/editors/task-editor/types.ts @@ -11,11 +11,13 @@ export interface Task { deleted?: boolean; position?: number; desc?: string; - due_date?: number + due_date?: number; done?: boolean; last_edited?: number; } +export type Align = "start" | "center" | "end" | "view" | false; + export type Headings = "Custom" | "Due" | "Changed"; export type HeadingsDir = "asc" | "desc"; diff --git a/src/packages/frontend/frame-editors/task-editor/actions.ts b/src/packages/frontend/frame-editors/task-editor/actions.ts index a1e8f1fea4..308bb84fc2 100644 --- a/src/packages/frontend/frame-editors/task-editor/actions.ts +++ b/src/packages/frontend/frame-editors/task-editor/actions.ts @@ -17,11 +17,15 @@ import { TaskStore } from "@cocalc/frontend/editors/task-editor/store"; import { redux_name } from "../../app-framework"; import { aux_file, cmp } from "@cocalc/util/misc"; import { Map } from "immutable"; +import { delay } from "awaiting"; +import type { FragmentId } from "@cocalc/frontend/misc/fragment-id"; interface TaskEditorState extends CodeEditorState { // nothing yet } +const FRAME_TYPE = "tasks"; + export class Actions extends CodeEditorActions { protected doctype: string = "syncdb"; protected primary_keys: string[] = ["task_id"]; @@ -39,7 +43,7 @@ export class Actions extends CodeEditorActions { this.auxPath = aux_file(this.path, "tasks"); this.taskStore = this.redux.createStore( redux_name(this.project_id, this.auxPath), - TaskStore + TaskStore, ); const syncdb = this._syncstring; syncdb.on("change", this.syncdbChange); @@ -136,7 +140,7 @@ export class Actions extends CodeEditorActions { this.auxPath, this._syncstring, this.taskStore, - this.path + this.path, ); actions._init_frame(frameId, this); this.taskActions[frameId] = actions; @@ -186,7 +190,7 @@ export class Actions extends CodeEditorActions { } _raw_default_frame_tree(): FrameTree { - return { type: "tasks" }; + return { type: FRAME_TYPE }; } async export_to_markdown(): Promise { @@ -201,7 +205,7 @@ export class Actions extends CodeEditorActions { if (id === undefined) { id = this._get_active_id(); } - if (this._get_frame_type(id) == "tasks") { + if (this._get_frame_type(id) == FRAME_TYPE) { this.getTaskActions(id)?.show(); return; } @@ -212,18 +216,18 @@ export class Actions extends CodeEditorActions { if (id === undefined) { id = this._get_active_id(); } - if (this._get_frame_type(id) == "tasks") { + if (this._get_frame_type(id) == FRAME_TYPE) { this.getTaskActions(id)?.hide(); } } protected languageModelGetText(frameId: string, scope): string { - if (this._get_frame_type(frameId) == "tasks") { + if (this._get_frame_type(frameId) == FRAME_TYPE) { const node = this._get_frame_node(frameId); return ( this.getTaskActions(frameId)?.chatgptGetText( scope, - node?.get("data-current_task_id") + node?.get("data-current_task_id"), ) ?? "" ); } @@ -238,8 +242,22 @@ export class Actions extends CodeEditorActions { return "md"; } - // async updateEmbeddings(): Promise { - // if (this._syncstring == null) return 0; - // return (await this.getTaskActions()?.updateEmbeddings()) ?? 0; - // } + async gotoFragment(fragmentId: FragmentId) { + const { id } = fragmentId as any; + if (!id) { + return; + } + const frameId = await this.waitUntilFrameReady({ + type: FRAME_TYPE, + }); + if (!frameId) { + return; + } + for (const d of [1, 10, 50, 500, 1000]) { + const actions = this.getTaskActions(frameId); + actions?.set_current_task(id); + actions?.scrollIntoView("start"); + await delay(d); + } + } } From 814489d2b8a620ea0c071267297baf362e75fb6a Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 13 Oct 2024 23:29:36 +0000 Subject: [PATCH 13/16] search: working on refactoring to also support tasks --- .../frame-editors/chat-editor/actions.ts | 9 +++++ .../chat-editor/use-search-index.ts | 36 +++++++++++-------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/packages/frontend/frame-editors/chat-editor/actions.ts b/src/packages/frontend/frame-editors/chat-editor/actions.ts index 95ed85b7fc..077062594e 100644 --- a/src/packages/frontend/frame-editors/chat-editor/actions.ts +++ b/src/packages/frontend/frame-editors/chat-editor/actions.ts @@ -23,6 +23,7 @@ import { redux_name } from "@cocalc/frontend/app-framework"; import { aux_file } from "@cocalc/util/misc"; import type { FragmentId } from "@cocalc/frontend/misc/fragment-id"; import { delay } from "awaiting"; +import { getSearchData } from "@cocalc/frontend/chat/filter-messages"; const FRAME_TYPE = "chatroom"; @@ -157,4 +158,12 @@ export class Actions extends CodeEditorActions { await delay(d); } } + + getSearchData = () => { + const messages = this.store?.get("messages"); + if (messages == null) { + return {}; + } + return getSearchData({ messages, threads: false }); + }; } diff --git a/src/packages/frontend/frame-editors/chat-editor/use-search-index.ts b/src/packages/frontend/frame-editors/chat-editor/use-search-index.ts index 63a9ff36a3..addc2c35cd 100644 --- a/src/packages/frontend/frame-editors/chat-editor/use-search-index.ts +++ b/src/packages/frontend/frame-editors/chat-editor/use-search-index.ts @@ -1,17 +1,27 @@ import { useFrameContext } from "@cocalc/frontend/frame-editors/frame-tree/frame-context"; -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { create, search, insertMultiple } from "@orama/orama"; -import { getSearchData } from "@cocalc/frontend/chat/filter-messages"; import useCounter from "@cocalc/frontend/app-framework/counter-hook"; export default function useSearchIndex() { const { actions, project_id, path } = useFrameContext(); + const contextRef = useRef<{ project_id: string; path: string }>({ + project_id, + path, + }); const [index, setIndex] = useState(null); const [error, setError] = useState(""); const { val: refresh, inc: doRefresh } = useCounter(); const [indexTime, setIndexTime] = useState(0); useEffect(() => { + if ( + contextRef.current.project_id != project_id || + contextRef.current.path != path + ) { + contextRef.current = { project_id, path }; + setIndex(null); + } (async () => { try { setError(""); @@ -57,19 +67,17 @@ class SearchIndex { }, }); - const messages = this.actions.store?.get("messages"); - if (messages == null) { - return; - } - const searchData = getSearchData({ messages, threads: false }); - const docs: { time: number; message: string }[] = []; - for (const time in searchData) { - docs.push({ - time: parseInt(time), - message: searchData[time]?.content ?? "", - }); + const searchData = this.actions.getSearchData(); + if (searchData != null) { + const docs: { time: number; message: string }[] = []; + for (const time in searchData) { + docs.push({ + time: parseInt(time), + message: searchData[time]?.content ?? "", + }); + } + await insertMultiple(this.db, docs); } - await insertMultiple(this.db, docs); this.state = "ready"; }; } From ea80e0a2c94f9259212c799926eeef6252ecabde Mon Sep 17 00:00:00 2001 From: William Stein Date: Sun, 13 Oct 2024 23:56:53 +0000 Subject: [PATCH 14/16] search: making it more generic -- the fragment --- src/packages/frontend/chat/actions.ts | 2 + .../frame-editors/chat-editor/actions.ts | 9 +++- .../frame-editors/chat-editor/search.tsx | 17 +++--- .../chat-editor/use-search-index.ts | 52 +++++++++++-------- 4 files changed, 50 insertions(+), 30 deletions(-) diff --git a/src/packages/frontend/chat/actions.ts b/src/packages/frontend/chat/actions.ts index fb14d38fe3..f4c4657faa 100644 --- a/src/packages/frontend/chat/actions.ts +++ b/src/packages/frontend/chat/actions.ts @@ -517,12 +517,14 @@ export class ChatActions extends Actions { this.scrollToIndex(Number.MAX_SAFE_INTEGER); }; +// this scrolls the message with given date into view and sets it as the selected message. scrollToDate = (date) => { this.clearScrollRequest(); this.frameTreeActions?.set_frame_data({ id: this.frameId, fragmentId: toMsString(date), }); + this.setFragment(date); setTimeout(() => { this.frameTreeActions?.set_frame_data({ id: this.frameId, diff --git a/src/packages/frontend/frame-editors/chat-editor/actions.ts b/src/packages/frontend/frame-editors/chat-editor/actions.ts index 077062594e..e879da3d87 100644 --- a/src/packages/frontend/frame-editors/chat-editor/actions.ts +++ b/src/packages/frontend/frame-editors/chat-editor/actions.ts @@ -159,11 +159,16 @@ export class Actions extends CodeEditorActions { } } - getSearchData = () => { + getSearchIndexData = () => { const messages = this.store?.get("messages"); if (messages == null) { return {}; } - return getSearchData({ messages, threads: false }); + const data: { [id: string]: string } = {}; + const data0 = getSearchData({ messages, threads: false }); + for (const id in data0) { + data[id] = data0[id]?.content; + } + return { data, fragmentKey: "chat" }; }; } diff --git a/src/packages/frontend/frame-editors/chat-editor/search.tsx b/src/packages/frontend/frame-editors/chat-editor/search.tsx index 2815800c01..062d06cc87 100644 --- a/src/packages/frontend/frame-editors/chat-editor/search.tsx +++ b/src/packages/frontend/frame-editors/chat-editor/search.tsx @@ -42,7 +42,7 @@ function Search({ font_size, desc }: Props) { [project_id, path], ); - const { error, setError, index, doRefresh } = useSearchIndex(); + const { error, setError, index, doRefresh, fragmentKey } = useSearchIndex(); useEffect(() => { if (index == null) { @@ -69,7 +69,10 @@ function Search({ font_size, desc }: Props) {
Search Chatroom {separate_file_extension(path_split(path).tail).name} + <> + Search Chatroom{" "} + {separate_file_extension(path_split(path).tail).name} + } style={{ fontSize: font_size }} > @@ -93,7 +96,7 @@ function Search({ font_size, desc }: Props) {
{result?.hits?.map((hit) => ( - + ))} {result?.hits == null && search?.trim() &&
No hits
}
@@ -102,7 +105,7 @@ function Search({ font_size, desc }: Props) { ); } -function SearchResult({ hit, actions }) { +function SearchResult({ hit, actions, fragmentKey }) { const { document } = hit; return (
{ - actions.gotoFragment({ chat: document.time }); + actions.gotoFragment({ [fragmentKey]: document.id }); }} > - + */ }} />
diff --git a/src/packages/frontend/frame-editors/chat-editor/use-search-index.ts b/src/packages/frontend/frame-editors/chat-editor/use-search-index.ts index addc2c35cd..0f98ed4f9d 100644 --- a/src/packages/frontend/frame-editors/chat-editor/use-search-index.ts +++ b/src/packages/frontend/frame-editors/chat-editor/use-search-index.ts @@ -13,6 +13,7 @@ export default function useSearchIndex() { const [error, setError] = useState(""); const { val: refresh, inc: doRefresh } = useCounter(); const [indexTime, setIndexTime] = useState(0); + const [fragmentKey, setFragmentKey] = useState("id"); useEffect(() => { if ( @@ -26,34 +27,40 @@ export default function useSearchIndex() { try { setError(""); const t0 = Date.now(); - const index = new SearchIndex({ actions }); - await index.init(); - setIndex(index); + const newIndex = new SearchIndex({ actions }); + await newIndex.init(); + setFragmentKey(newIndex.fragmentKey ?? "id"); + setIndex(newIndex); setIndexTime(Date.now() - t0); + //index?.close(); } catch (err) { setError(`${err}`); } })(); }, [project_id, path, refresh]); - return { index, error, doRefresh, setError, indexTime }; + return { index, error, doRefresh, setError, indexTime, fragmentKey }; } class SearchIndex { - private actions; - private state: "init" | "ready" | "failed" = "init"; - private error: Error | null = null; - private db; + private actions?; + private state: "init" | "ready" | "failed" | "closed" = "init"; + private db?; + public fragmentKey?: string = "id"; constructor({ actions }) { this.actions = actions; } - getState = () => this.state; - getError = () => this.error; + close = () => { + this.state = "closed"; + delete this.actions; + delete this.db; + delete this.fragmentKey; + }; search = async (query) => { - if (this.state != "ready") { + if (this.state != "ready" || this.db == null) { throw Error("index not ready"); } return await search(this.db, query); @@ -62,19 +69,22 @@ class SearchIndex { init = async () => { this.db = await create({ schema: { - time: "number", - message: "string", + content: "string", }, }); - const searchData = this.actions.getSearchData(); - if (searchData != null) { - const docs: { time: number; message: string }[] = []; - for (const time in searchData) { - docs.push({ - time: parseInt(time), - message: searchData[time]?.content ?? "", - }); + if (this.actions == null || this.state != "init") { + throw Error("not in init state"); + } + const { data, fragmentKey } = this.actions.getSearchIndexData(); + this.fragmentKey = fragmentKey; + if (data != null) { + const docs: { id: string; content: string }[] = []; + for (const id in data) { + const content = data[id]?.trim(); + if (content) { + docs.push({ id, content }); + } } await insertMultiple(this.db, docs); } From 1789a189c8872afd72c7a45ec14f0bf5cc5f9186 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 14 Oct 2024 00:26:30 +0000 Subject: [PATCH 15/16] search: further refactoring to make it more generic --- .../frame-editors/chat-editor/search.tsx | 142 ++-------------- .../frame-editors/generic/search/index.tsx | 153 ++++++++++++++++++ .../search}/use-search-index.ts | 0 3 files changed, 163 insertions(+), 132 deletions(-) create mode 100644 src/packages/frontend/frame-editors/generic/search/index.tsx rename src/packages/frontend/frame-editors/{chat-editor => generic/search}/use-search-index.ts (100%) diff --git a/src/packages/frontend/frame-editors/chat-editor/search.tsx b/src/packages/frontend/frame-editors/chat-editor/search.tsx index 062d06cc87..1e082a837f 100644 --- a/src/packages/frontend/frame-editors/chat-editor/search.tsx +++ b/src/packages/frontend/frame-editors/chat-editor/search.tsx @@ -1,142 +1,20 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -Full text search that is better than a simple filter. -*/ - -import { useFrameContext } from "@cocalc/frontend/frame-editors/frame-tree/frame-context"; -import type { EditorDescription } from "@cocalc/frontend/frame-editors/frame-tree/types"; -import { Card, Input } from "antd"; -import { path_split, separate_file_extension, set } from "@cocalc/util/misc"; -import { useEffect, useMemo, useState } from "react"; -import { throttle } from "lodash"; -import useSearchIndex from "./use-search-index"; -import ShowError from "@cocalc/frontend/components/error"; import StaticMarkdown from "@cocalc/frontend/editors/slate/static-markdown"; import { TimeAgo } from "@cocalc/frontend/components"; -import { useEditorRedux } from "@cocalc/frontend/app-framework"; -import type { ChatState } from "@cocalc/frontend/chat/store"; - -interface Props { - font_size: number; - desc; -} - -function Search({ font_size, desc }: Props) { - const { project_id, path, actions, id } = useFrameContext(); - const useEditor = useEditorRedux({ project_id, path }); - const messages = useEditor("messages"); - const [indexedMessages, setIndexedMessages] = useState(messages); - const [search, setSearch] = useState(desc.get("data-search") ?? ""); - const [result, setResult] = useState(null); - const saveSearch = useMemo( - () => - throttle((search) => { - if (!actions.isClosed()) { - actions.set_frame_data({ id, search }); - } - }, 250), - [project_id, path], - ); - - const { error, setError, index, doRefresh, fragmentKey } = useSearchIndex(); - - useEffect(() => { - if (index == null) { - return; - } - if (!search.trim()) { - setResult([]); - return; - } - (async () => { - const result = await index.search({ term: search }); - setResult(result); - })(); - }, [search, index]); - - useEffect(() => { - if (indexedMessages != messages) { - setIndexedMessages(messages); - doRefresh(); - } - }, [messages]); +import { createSearchEditor } from "@cocalc/frontend/frame-editors/generic/search"; +function Preview({ id, content }) { return ( -
- - Search Chatroom{" "} - {separate_file_extension(path_split(path).tail).name} - - } - style={{ fontSize: font_size }} - > - - { - const search = e.target.value ?? ""; - setSearch(search); - saveSearch(search); - }} - /> - -
-
- {result?.hits?.map((hit) => ( - - ))} - {result?.hits == null && search?.trim() &&
No hits
} -
-
-
- ); -} - -function SearchResult({ hit, actions, fragmentKey }) { - const { document } = hit; - return ( -
{ - actions.gotoFragment({ [fragmentKey]: document.id }); - }} - > - + <> + */ }} /> -
+ ); } -export const search = { - type: "search", - short: "Search", - name: "Search", - icon: "comment", - commands: set(["decrease_font_size", "increase_font_size"]), - component: Search, -} as EditorDescription; +export const search = createSearchEditor({ Preview }); diff --git a/src/packages/frontend/frame-editors/generic/search/index.tsx b/src/packages/frontend/frame-editors/generic/search/index.tsx new file mode 100644 index 0000000000..654d9ec502 --- /dev/null +++ b/src/packages/frontend/frame-editors/generic/search/index.tsx @@ -0,0 +1,153 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +/* +Full text search that is better than a simple filter. +*/ + +import { useFrameContext } from "@cocalc/frontend/frame-editors/frame-tree/frame-context"; +import type { EditorDescription } from "@cocalc/frontend/frame-editors/frame-tree/types"; +import { Card, Input } from "antd"; +import { path_split, separate_file_extension, set } from "@cocalc/util/misc"; +import { useEffect, useMemo, useState } from "react"; +import { throttle } from "lodash"; +import useSearchIndex from "./use-search-index"; +import ShowError from "@cocalc/frontend/components/error"; +import { useEditorRedux } from "@cocalc/frontend/app-framework"; +import type { ChatState } from "@cocalc/frontend/chat/store"; + +interface Props { + font_size: number; + desc; + Preview?; +} + +function Search({ font_size, desc, Preview }: Props) { + const { project_id, path, actions, id } = useFrameContext(); + const useEditor = useEditorRedux({ project_id, path }); + const messages = useEditor("messages"); + const [indexedMessages, setIndexedMessages] = useState(messages); + const [search, setSearch] = useState(desc.get("data-search") ?? ""); + const [result, setResult] = useState(null); + const saveSearch = useMemo( + () => + throttle((search) => { + if (!actions.isClosed()) { + actions.set_frame_data({ id, search }); + } + }, 250), + [project_id, path], + ); + + const { error, setError, index, doRefresh, fragmentKey } = useSearchIndex(); + + useEffect(() => { + if (index == null) { + return; + } + if (!search.trim()) { + setResult([]); + return; + } + (async () => { + const result = await index.search({ term: search }); + setResult(result); + })(); + }, [search, index]); + + useEffect(() => { + if (indexedMessages != messages) { + setIndexedMessages(messages); + doRefresh(); + } + }, [messages]); + + return ( +
+ + Search Chatroom{" "} + {separate_file_extension(path_split(path).tail).name} + + } + style={{ fontSize: font_size }} + > + + { + const search = e.target.value ?? ""; + setSearch(search); + saveSearch(search); + }} + /> + +
+
+ {result?.hits?.map((hit) => ( + + ))} + {result?.hits == null && search?.trim() &&
No hits
} +
+
+
+ ); +} + +function SearchResult({ hit, actions, fragmentKey, Preview }) { + const { document } = hit; + return ( +
{ + actions.gotoFragment({ [fragmentKey]: document.id }); + }} + > + {Preview != null ? ( + + ) : ( +
{document.content}
+ )} +
+ ); +} + +export function createSearchEditor({ + Preview, +}: { + Preview?; +}): EditorDescription { + return { + type: "search", + short: "Search", + name: "Search", + icon: "comment", + commands: set(["decrease_font_size", "increase_font_size"]), + component: (props) => , + } as EditorDescription; +} diff --git a/src/packages/frontend/frame-editors/chat-editor/use-search-index.ts b/src/packages/frontend/frame-editors/generic/search/use-search-index.ts similarity index 100% rename from src/packages/frontend/frame-editors/chat-editor/use-search-index.ts rename to src/packages/frontend/frame-editors/generic/search/use-search-index.ts From fddee81ea906ca713175156d7ad76ed9538686e6 Mon Sep 17 00:00:00 2001 From: William Stein Date: Mon, 14 Oct 2024 01:41:24 +0000 Subject: [PATCH 16/16] make search also work for tasks --- .../frontend/editors/task-editor/actions.ts | 9 +- .../frontend/editors/task-editor/editor.tsx | 209 +++++++++--------- .../frontend/editors/task-editor/find.tsx | 3 +- .../frontend/editors/task-editor/store.ts | 2 +- .../frontend/editors/task-editor/task.tsx | 5 +- .../frame-editors/chat-editor/search.tsx | 6 +- .../frame-editors/generic/search/index.tsx | 106 ++++++--- .../frame-editors/task-editor/actions.ts | 63 ++++-- .../frame-editors/task-editor/editor.ts | 11 +- .../frame-editors/task-editor/search.tsx | 22 ++ 10 files changed, 280 insertions(+), 156 deletions(-) create mode 100644 src/packages/frontend/frame-editors/task-editor/search.tsx diff --git a/src/packages/frontend/editors/task-editor/actions.ts b/src/packages/frontend/editors/task-editor/actions.ts index f70dc86cfa..6c7309aafe 100644 --- a/src/packages/frontend/editors/task-editor/actions.ts +++ b/src/packages/frontend/editors/task-editor/actions.ts @@ -34,10 +34,12 @@ import { TaskMap, TaskState, } from "./types"; -import { TaskStore } from "./store"; import { SyncDB } from "@cocalc/sync/editor/db"; import { webapp_client } from "../../webapp-client"; -import type { Actions as TaskFrameActions } from "@cocalc/frontend/frame-editors/task-editor/actions"; +import type { + Actions as TaskFrameActions, + Store as TaskStore, +} from "@cocalc/frontend/frame-editors/task-editor/actions"; import Fragment from "@cocalc/frontend/misc/fragment-id"; const LAST_EDITED_THRESH_S = 30; @@ -48,7 +50,7 @@ export class TaskActions extends Actions { private project_id: string; private path: string; private truePath: string; - private store: TaskStore; + public store: TaskStore; _update_visible: Function; private is_closed: boolean = false; private key_handler?: (any) => void; @@ -663,7 +665,6 @@ export class TaskActions extends Actions { } public blur_find_box(): void { - this.enable_key_handler(); this.setFrameData({ focus_find_box: false }); } diff --git a/src/packages/frontend/editors/task-editor/editor.tsx b/src/packages/frontend/editors/task-editor/editor.tsx index 1a9c7b6a1e..3890361670 100644 --- a/src/packages/frontend/editors/task-editor/editor.tsx +++ b/src/packages/frontend/editors/task-editor/editor.tsx @@ -9,9 +9,8 @@ Top-level react component for task list import { Button } from "antd"; import { fromJS } from "immutable"; - import { Col, Row } from "@cocalc/frontend/antd-bootstrap"; -import { React, useEditorRedux } from "@cocalc/frontend/app-framework"; +import { useEditorRedux } from "@cocalc/frontend/app-framework"; import { Loading } from "@cocalc/frontend/components"; import { Icon } from "@cocalc/frontend/components/icon"; import { TaskActions } from "./actions"; @@ -31,111 +30,115 @@ interface Props { read_only?: boolean; } -export const TaskEditor: React.FC = React.memo( - ({ actions, path, project_id, desc, read_only }) => { - const useEditor = useEditorRedux({ project_id, path }); - const tasks = useEditor("tasks"); - const visible = desc.get("data-visible"); - const local_task_state = desc.get("data-local_task_state") ?? fromJS({}); - const local_view_state = desc.get("data-local_view_state") ?? fromJS({}); - const hashtags = desc.get("data-hashtags"); - const current_task_id = desc.get("data-current_task_id"); - const counts = desc.get("data-counts"); - const search_terms = desc.get("data-search_terms"); - const search_desc = desc.get("data-search_desc"); - const focus_find_box = desc.get("data-focus_find_box"); +export function TaskEditor({ + actions, + path, + project_id, + desc, + read_only, +}: Props) { + const useEditor = useEditorRedux({ project_id, path }); + const tasks = useEditor("tasks"); + const visible = desc.get("data-visible"); + const local_task_state = desc.get("data-local_task_state") ?? fromJS({}); + const local_view_state = desc.get("data-local_view_state") ?? fromJS({}); + const hashtags = desc.get("data-hashtags"); + const current_task_id = desc.get("data-current_task_id"); + const counts = desc.get("data-counts"); + const search_terms = desc.get("data-search_terms"); + const search_desc = desc.get("data-search_desc"); + const focus_find_box = desc.get("data-focus_find_box"); - if (tasks == null || visible == null) { - return ( -
+ +
+ ); + } + + return ( +
+ + + + + + + + + + + + {hashtags != null && ( + + )} + + + +
+ {visible.size == 0 ? ( + - -
- ); - } - - return ( - + ); +} diff --git a/src/packages/frontend/editors/task-editor/find.tsx b/src/packages/frontend/editors/task-editor/find.tsx index 07c0c9e626..5f0a5517f5 100644 --- a/src/packages/frontend/editors/task-editor/find.tsx +++ b/src/packages/frontend/editors/task-editor/find.tsx @@ -44,7 +44,7 @@ export function Find({ ref={inputRef} allowClear value={local_view_state.get("search") ?? ""} - placeholder={"Search for tasks..."} + placeholder={"Filter tasks..."} onChange={(e) => actions.set_local_view_state({ search: e.target.value, @@ -56,6 +56,7 @@ export function Find({ if (evt.which === 27) { actions.set_local_view_state({ search: "" }); inputRef.current?.blur(); + actions.enable_key_handler(); return false; } }} diff --git a/src/packages/frontend/editors/task-editor/store.ts b/src/packages/frontend/editors/task-editor/store.ts index 4be243db1e..8b871c348d 100644 --- a/src/packages/frontend/editors/task-editor/store.ts +++ b/src/packages/frontend/editors/task-editor/store.ts @@ -4,6 +4,6 @@ */ import { Store } from "../../app-framework"; -import { TaskState } from "./types"; +import type { TaskState } from "./types"; export class TaskStore extends Store {} diff --git a/src/packages/frontend/editors/task-editor/task.tsx b/src/packages/frontend/editors/task-editor/task.tsx index e223d4fb85..76d0f9318e 100644 --- a/src/packages/frontend/editors/task-editor/task.tsx +++ b/src/packages/frontend/editors/task-editor/task.tsx @@ -99,7 +99,10 @@ export default function Task({ return ( actions?.set_current_task(task.get("task_id"))} + onClick={() => { + actions?.set_current_task(task.get("task_id")); + actions?.enable_key_handler(); + }} > diff --git a/src/packages/frontend/frame-editors/chat-editor/search.tsx b/src/packages/frontend/frame-editors/chat-editor/search.tsx index 1e082a837f..f0fc472ccc 100644 --- a/src/packages/frontend/frame-editors/chat-editor/search.tsx +++ b/src/packages/frontend/frame-editors/chat-editor/search.tsx @@ -17,4 +17,8 @@ function Preview({ id, content }) { ); } -export const search = createSearchEditor({ Preview }); +export const search = createSearchEditor({ + Preview, + updateField: "messages", + title: "Chatroom", +}); diff --git a/src/packages/frontend/frame-editors/generic/search/index.tsx b/src/packages/frontend/frame-editors/generic/search/index.tsx index 654d9ec502..71b5707da0 100644 --- a/src/packages/frontend/frame-editors/generic/search/index.tsx +++ b/src/packages/frontend/frame-editors/generic/search/index.tsx @@ -16,18 +16,59 @@ import { throttle } from "lodash"; import useSearchIndex from "./use-search-index"; import ShowError from "@cocalc/frontend/components/error"; import { useEditorRedux } from "@cocalc/frontend/app-framework"; -import type { ChatState } from "@cocalc/frontend/chat/store"; + +export function createSearchEditor({ + Preview, + updateField, + previewStyle, +}: { + // component for previewing search results. + Preview?; + // name of a field in the store so that we should update the search index + // exactly when the value of that field changes. + updateField: string; + // overload styles for component that contains the preview, e.g., maxHeight could be made bigger. + previewStyle?; + title?: string; +}): EditorDescription { + return { + type: "search", + short: "Search", + name: "Search", + icon: "comment", + commands: set(["decrease_font_size", "increase_font_size"]), + component: (props) => ( + + ), + } as EditorDescription; +} interface Props { font_size: number; desc; + updateField: string; Preview?; + previewStyle?; + title?; } -function Search({ font_size, desc, Preview }: Props) { +function Search({ + font_size: fontSize, + desc, + Preview, + updateField, + previewStyle, + title, +}: Props) { const { project_id, path, actions, id } = useFrameContext(); - const useEditor = useEditorRedux({ project_id, path }); - const messages = useEditor("messages"); + const useEditor = useEditorRedux({ project_id, path }); + // @ts-ignore + const messages = useEditor(updateField); const [indexedMessages, setIndexedMessages] = useState(messages); const [search, setSearch] = useState(desc.get("data-search") ?? ""); const [result, setResult] = useState(null); @@ -52,7 +93,7 @@ function Search({ font_size, desc, Preview }: Props) { return; } (async () => { - const result = await index.search({ term: search }); + const result = await index.search({ term: search, limit: 30 /* todo */ }); setResult(result); })(); }, [search, index]); @@ -69,19 +110,18 @@ function Search({ font_size, desc, Preview }: Props) { - Search Chatroom{" "} - {separate_file_extension(path_split(path).tail).name} + Search {title} {separate_file_extension(path_split(path).tail).name} } - style={{ fontSize: font_size }} + style={{ fontSize }} >
+
+ {!search?.trim() && Enter a search above} + {(result?.hits?.length ?? 0) == 0 && search?.trim() && ( + No Matches + )} + {(result?.count ?? 0) > (result?.hits?.length ?? 0) && ( + + Showing {result?.hits.length} of {result?.count ?? 0} results + + )} +
{result?.hits?.map((hit) => ( ))} - {result?.hits == null && search?.trim() &&
No hits
}
); } -function SearchResult({ hit, actions, fragmentKey, Preview }) { +function SearchResult({ + hit, + actions, + fragmentKey, + Preview, + previewStyle, + fontSize, +}) { const { document } = hit; return (
{ actions.gotoFragment({ [fragmentKey]: document.id }); }} > {Preview != null ? ( - + ) : (
{document.content}
)}
); } - -export function createSearchEditor({ - Preview, -}: { - Preview?; -}): EditorDescription { - return { - type: "search", - short: "Search", - name: "Search", - icon: "comment", - commands: set(["decrease_font_size", "increase_font_size"]), - component: (props) => , - } as EditorDescription; -} diff --git a/src/packages/frontend/frame-editors/task-editor/actions.ts b/src/packages/frontend/frame-editors/task-editor/actions.ts index 308bb84fc2..2ea4c3e1f9 100644 --- a/src/packages/frontend/frame-editors/task-editor/actions.ts +++ b/src/packages/frontend/frame-editors/task-editor/actions.ts @@ -13,19 +13,23 @@ import { } from "../code-editor/actions"; import { FrameTree } from "../frame-tree/types"; import { TaskActions } from "@cocalc/frontend/editors/task-editor/actions"; -import { TaskStore } from "@cocalc/frontend/editors/task-editor/store"; -import { redux_name } from "../../app-framework"; +import { redux_name } from "@cocalc/frontend/app-framework"; +import type { Store as BaseStore } from "@cocalc/frontend/app-framework"; import { aux_file, cmp } from "@cocalc/util/misc"; import { Map } from "immutable"; import { delay } from "awaiting"; import type { FragmentId } from "@cocalc/frontend/misc/fragment-id"; +import type { Tasks } from "@cocalc/frontend/editors/task-editor/types"; +import { DONE } from "./search"; interface TaskEditorState extends CodeEditorState { - // nothing yet + tasks?: Tasks; } const FRAME_TYPE = "tasks"; +export type Store = BaseStore; + export class Actions extends CodeEditorActions { protected doctype: string = "syncdb"; protected primary_keys: string[] = ["task_id"]; @@ -36,15 +40,10 @@ export class Actions extends CodeEditorActions { metaColumns: ["due_date", "done"], }; taskActions: { [frameId: string]: TaskActions } = {}; - taskStore: TaskStore; auxPath: string; _init2(): void { this.auxPath = aux_file(this.path, "tasks"); - this.taskStore = this.redux.createStore( - redux_name(this.project_id, this.auxPath), - TaskStore, - ); const syncdb = this._syncstring; syncdb.on("change", this.syncdbChange); syncdb.once("change", this.ensurePositionsAreUnique); @@ -52,7 +51,7 @@ export class Actions extends CodeEditorActions { private syncdbChange(changes) { const syncdb = this._syncstring; - const store = this.taskStore; + const store = this.store; if (syncdb == null || store == null) { // may happen during close return; @@ -77,7 +76,7 @@ export class Actions extends CodeEditorActions { } private ensurePositionsAreUnique() { - let tasks = this.taskStore.get("tasks"); + let tasks = this.store.get("tasks"); if (tasks == null) { return; } @@ -139,11 +138,12 @@ export class Actions extends CodeEditorActions { this.project_id, this.auxPath, this._syncstring, - this.taskStore, + this.store, this.path, ); actions._init_frame(frameId, this); this.taskActions[frameId] = actions; + actions.store = this.store; return actions; } @@ -185,7 +185,7 @@ export class Actions extends CodeEditorActions { for (const frameId in this.taskActions) { this.closeTaskFrame(frameId); } - this.redux.removeStore(this.taskStore.name); + this.redux.removeStore(this.store.name); super.close(); } @@ -201,14 +201,23 @@ export class Actions extends CodeEditorActions { } } + private hideAllTaskActionsExcept = (id) => { + for (const id0 in this.taskActions) { + if (id0 != id) { + this.taskActions[id0].hide(); + } + } + }; + public focus(id?: string): void { if (id === undefined) { id = this._get_active_id(); } - if (this._get_frame_type(id) == FRAME_TYPE) { - this.getTaskActions(id)?.show(); - return; - } + this.hideAllTaskActionsExcept(id); + // if (this._get_frame_type(id) == FRAME_TYPE) { + // this.getTaskActions(id)?.show(); + // return; + // } super.focus(id); } @@ -260,4 +269,26 @@ export class Actions extends CodeEditorActions { await delay(d); } } + + getSearchIndexData = () => { + const tasks = this.store?.get("tasks"); + if (tasks == null) { + return {}; + } + const data: { [id: string]: string } = {}; + for (const [id, task] of tasks) { + if (task.get("deleted")) { + continue; + } + let content = task.get("desc")?.trim(); + if (!content) { + continue; + } + if (task.get("done")) { + content = DONE + content; + } + data[id] = content; + } + return { data, fragmentKey: "id" }; + }; } diff --git a/src/packages/frontend/frame-editors/task-editor/editor.ts b/src/packages/frontend/frame-editors/task-editor/editor.ts index 2581c3ca87..d39fdeb02a 100644 --- a/src/packages/frontend/frame-editors/task-editor/editor.ts +++ b/src/packages/frontend/frame-editors/task-editor/editor.ts @@ -14,6 +14,7 @@ import { createEditor } from "../frame-tree/editor"; import { EditorDescription } from "../frame-tree/types"; import { terminal } from "../terminal-editor/editor"; import { time_travel } from "../time-travel-editor/editor"; +import { search } from "./search"; const tasks: EditorDescription = { type: "tasks", @@ -25,7 +26,6 @@ const tasks: EditorDescription = { return createElement(TaskEditor, { ...props, actions, - path: actions.path, }); }, commands: set([ @@ -38,6 +38,14 @@ const tasks: EditorDescription = { "help", "export_to_markdown", "chatgpt", + "show_search", + ]), + buttons: set([ + "undo", + "redo", + "decrease_font_size", + "increase_font_size", + "show_search", ]), } as const; @@ -45,6 +53,7 @@ const EDITOR_SPEC = { tasks, terminal, time_travel, + search, } as const; export const Editor = createEditor({ diff --git a/src/packages/frontend/frame-editors/task-editor/search.tsx b/src/packages/frontend/frame-editors/task-editor/search.tsx new file mode 100644 index 0000000000..ec7966ca31 --- /dev/null +++ b/src/packages/frontend/frame-editors/task-editor/search.tsx @@ -0,0 +1,22 @@ +import StaticMarkdown from "@cocalc/frontend/editors/slate/static-markdown"; +import { createSearchEditor } from "@cocalc/frontend/frame-editors/generic/search"; + +export const DONE = "☑ "; + +function Preview({ content }) { + return ( + */, + opacity: content.startsWith(DONE) ? 0.5 : undefined, + }} + /> + ); +} + +export const search = createSearchEditor({ + Preview, + updateField: "tasks", + title: "Task List", +});