From a15532df50c0b7f0075df257f47a30b79b3015c4 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 27 Feb 2024 01:43:52 +0000 Subject: [PATCH 1/5] salesloft -- need to nest custom fields --- src/packages/server/salesloft/money.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/packages/server/salesloft/money.ts b/src/packages/server/salesloft/money.ts index 7cd6f660c6..7a573477b9 100644 --- a/src/packages/server/salesloft/money.ts +++ b/src/packages/server/salesloft/money.ts @@ -79,10 +79,12 @@ export async function getMoneyData(account_id: string): Promise<{ if (x.rows.length == 0) { // no statements ever return { - cocalc_balance: 0, - cocalc_purchase_timestamp: "0000-00-00T00:00:00.000Z", - cocalc_last_month_spend: 0, - cocalc_last_year_spend: 0, + custom_fields: { + cocalc_balance: 0, + cocalc_purchase_timestamp: "0000-00-00T00:00:00.000Z", + cocalc_last_month_spend: 0, + cocalc_last_year_spend: 0, + }, }; } const cocalc_balance = x.rows[0].balance; From 8df1d0551659a31501c08129823ceac508075102 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 27 Feb 2024 01:51:21 +0000 Subject: [PATCH 2/5] salesloft -- properly fix adding custom fields (move to the right level) --- src/packages/server/salesloft/money.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/packages/server/salesloft/money.ts b/src/packages/server/salesloft/money.ts index 7a573477b9..76c4ba8247 100644 --- a/src/packages/server/salesloft/money.ts +++ b/src/packages/server/salesloft/money.ts @@ -59,7 +59,7 @@ export async function updateMoney(cutoff: string = "2 days") { } const data = await getMoneyData(account_id); log.debug("updateMoney: ", { salesloft_id: id, account_id, data }); - await update(id, data); + await update(id, { custom_fields: data }); } } @@ -79,12 +79,10 @@ export async function getMoneyData(account_id: string): Promise<{ if (x.rows.length == 0) { // no statements ever return { - custom_fields: { - cocalc_balance: 0, - cocalc_purchase_timestamp: "0000-00-00T00:00:00.000Z", - cocalc_last_month_spend: 0, - cocalc_last_year_spend: 0, - }, + cocalc_balance: 0, + cocalc_purchase_timestamp: "0000-00-00T00:00:00.000Z", + cocalc_last_month_spend: 0, + cocalc_last_year_spend: 0, }; } const cocalc_balance = x.rows[0].balance; From 578dd5e408e55f4eb58105e85a101efd75d10733 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 27 Feb 2024 02:00:04 +0000 Subject: [PATCH 3/5] fix share server link --- .../landing/cocalc-com-features.tsx | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/packages/next/components/landing/cocalc-com-features.tsx b/src/packages/next/components/landing/cocalc-com-features.tsx index a4cab26136..899ab43aca 100644 --- a/src/packages/next/components/landing/cocalc-com-features.tsx +++ b/src/packages/next/components/landing/cocalc-com-features.tsx @@ -59,9 +59,8 @@ export function CoCalcComFeatures() { belowWide={true} > - With {siteName}, you can easily collaborate with colleagues, - students, and friends to edit computational documents. We support - {" "} + With {siteName}, you can easily collaborate with colleagues, students, + and friends to edit computational documents. We support{" "} Jupyter Notebooks @@ -72,18 +71,16 @@ export function CoCalcComFeatures() { - Everyone's code runs in the same per-project environment, which provides - consistent results, synchronized file changes, and automatic revision - history so that you can go back in time when you need to discover what - changed and when. {shareServer && renderShareServer()} + Everyone's code runs in the same per-project environment, which + provides consistent results, synchronized file changes, and automatic + revision history so that you can go back in time when you need to + discover what changed and when. {renderShareServer()} Forget the frustration of sending files back and forth between your - collaborators, wasting time reviewing changes, and merging documents. {" "} - - Get started with {siteName} today. - + collaborators, wasting time reviewing changes, and merging documents.{" "} + Get started with {siteName} today. ); @@ -162,12 +159,9 @@ export function CoCalcComFeatures() { return ( <> - { " " }You can even publish your { siteName } creations to share with - anyone via the built-in { " " } - - share server - - . + {" "} + You can even publish your {siteName} creations to share with anyone via + the built-in share server. ); } From 9ae560aef7e13550e4400071d7f1113392a23548 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 27 Feb 2024 04:51:17 +0000 Subject: [PATCH 4/5] remove deprecated jupyter browser api call --- src/packages/project/browser-websocket/api.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts index de775bee95..1fbcb87392 100644 --- a/src/packages/project/browser-websocket/api.ts +++ b/src/packages/project/browser-websocket/api.ts @@ -134,11 +134,6 @@ async function handleApiCall(data: Mesg, spark): Promise { case "prettier_string": // deprecated case "formatter_string": return await run_formatter_string(data.path, data.str, data.options, log); - case "jupyter": - // DEPRECATED: The "jupyter" endpoint is only here for browser client - // backward compatibility. Can be safely deleted soon, but not immediately - // to make the release easier - return await jupyter(data.path, data.endpoint, data.query); case "exec": if (data.opts?.compute_server_id) { if (data.opts.filesystem) { @@ -222,8 +217,6 @@ async function listing( } } -import { handleApiRequest as jupyter } from "@cocalc/jupyter/kernel/websocket-api"; - // Execute code import { executeCode } from "@cocalc/backend/execute-code"; From 30294947fff662fc5b45145baa0c4f3cbd2fb807 Mon Sep 17 00:00:00 2001 From: William Stein Date: Tue, 27 Feb 2024 05:55:33 +0000 Subject: [PATCH 5/5] instead of relying on autosave, make it so whenever any browser client disconnects from a file editing session, that file gets saved to disk - hard case is jupyter due to the syncdb v ipynb file format complexity - autosave doesn't fully work anymore, is complicated, and might fail (e.g., if project stops), hence motivation for this --- src/packages/jupyter/kernel/kernel.ts | 7 +++ src/packages/project/sync/server.ts | 90 +++++++++++++++++---------- src/packages/project/sync/sync-doc.ts | 5 ++ src/packages/util/smc-version.js | 2 +- 4 files changed, 69 insertions(+), 35 deletions(-) diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index 48f380a224..7d8a452c78 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -152,6 +152,13 @@ export async function initJupyterRedux(syncdb: SyncDB, client: Client) { ); } +export async function getJupyterRedux(syncdb: SyncDB) { + const project_id = syncdb.project_id; + const path = original_path(syncdb.get_path()); + const name = redux_name(project_id, path); + return { actions: redux.getActions(name), store: redux.getStore(name) }; +} + // Remove the store/actions for a given Jupyter notebook, // and also close the kernel if it is running. export async function removeJupyterRedux( diff --git a/src/packages/project/sync/server.ts b/src/packages/project/sync/server.ts index 8eca3a7f75..2ade893ba3 100644 --- a/src/packages/project/sync/server.ts +++ b/src/packages/project/sync/server.ts @@ -21,11 +21,6 @@ silently swallowed in persistent mode... // and https://github.com/sagemathinc/cocalc/issues/5823 // and https://github.com/sagemathinc/cocalc/issues/5617 -// Setting this to 0 to optimize resource usage and because opening files -// is fast, and also on the current tab gets opened on refresh anyways. - -const CLOSE_DELAY_MS = 0; - // This is a hard upper bound on the number of browser sessions that could // have the same file open at once. We put some limit on it, to at least // limit problems from bugs which crash projects (since each connection uses @@ -56,7 +51,7 @@ import { // @ts-ignore -- typescript nonsense. const _ = set_debug; -import { init_syncdoc } from "./sync-doc"; +import { init_syncdoc, getSyncDocFromSyncTable } from "./sync-doc"; import { key, register_synctable } from "./open-synctables"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { once } from "@cocalc/util/async-utils"; @@ -68,6 +63,7 @@ import { register_project_status_table } from "./project-status"; import { register_usage_info_table } from "./usage-info"; import type { MergeType } from "@cocalc/sync/table/synctable"; import Client from "@cocalc/sync-client"; +import { getJupyterRedux } from "@cocalc/jupyter/kernel"; type Query = { [key: string]: any }; @@ -373,7 +369,7 @@ class SyncTableChannel { } } - this.check_if_should_close(); + this.check_if_should_save_or_close(); } private send_synctable_to_browser(spark: Spark): void { @@ -389,18 +385,20 @@ class SyncTableChannel { this.channel.write(x); } - /* Check if we should close, e.g., due to no connected clients. */ - private check_if_should_close(): void { - if (this.closed || this.persistent) { - // don't bother if either already closed, or the persistent option is set. + /* This is called when a user disconnects. This always triggers a save to + disk. It may also trigger closing the file in some cases. */ + private async check_if_should_save_or_close() { + if (this.closed) { + // don't bother if either already closed return; } - const { n } = this.num_connections; - if (n === 0) { - this.log("check_if_should_close -- ", n, " -- do a save and maybe close"); - this.save_and_close_if_possible(); - } else { - this.log("check_if_should_close -- ", n, " -- do not close"); + this.log("check_if_should_save_or_close: save to disk if possible"); + await this.save_if_possible(); + const { n } = this.num_connections ?? {}; + this.log("check_if_should_save_or_close", { n }); + if (!this.persistent && n === 0) { + this.log("check_if_should_save_or_close: close if possible"); + await this.close_if_possible(); } } @@ -450,33 +448,57 @@ class SyncTableChannel { this.channel.write(x); } - private async save_and_close_if_possible(): Promise { + private async save_if_possible(): Promise { if (this.closed || this.closing) { return; // closing or already closed } - this.log("save_and_close_if_possible: no connections, so saving..."); + this.log("save_if_possible: saves changes to database"); await this.synctable.save(); + if (this.synctable.table === "syncstrings") { + this.log("save_if_possible: also fetch syncdoc"); + const syncdoc = getSyncDocFromSyncTable(this.synctable); + if (syncdoc != null) { + const path = syncdoc.get_path(); + this.log("save_if_possible: saving syncdoc to disk", { path }); + if (path.endsWith(".sage-jupyter2")) { + // treat jupyter notebooks in a special way, since they have + // an aux .ipynb file that the syncdoc doesn't know about. In + // this case we save the ipynb to disk, not just the hidden + // syncdb file. + const { actions } = await getJupyterRedux(syncdoc); + if (actions == null) { + this.log("save_if_possible: jupyter -- actions is null"); + } else { + if (!actions.isCellRunner()) { + this.log("save_if_possible: jupyter -- not cell runner"); + return; + } + this.log("save_if_possible: jupyter -- saving to ipynb"); + await actions.save_ipynb_file(); + } + } + await syncdoc.save_to_disk(); + } else { + this.log("save_if_possible: no syncdoc"); + } + } + } + + private async close_if_possible(): Promise { + if (this.closed || this.closing) { + return; // closing or already closed + } const { n, changed } = this.num_connections; const delay = Date.now() - changed.valueOf(); this.log( - `save_and_close_if_possible: after save there are ${n} connections and delay=${delay}`, + `close_if_possible: there are ${n} connections and delay=${delay}`, ); if (n === 0) { - if (delay < CLOSE_DELAY_MS) { - this.log(`save_and_close_if_possible: wait a bit then try again`); - setTimeout( - this.check_if_should_close.bind(this), - 1000 + CLOSE_DELAY_MS - delay, - ); - } else { - this.log( - `save_and_close_if_possible: close this SyncTableChannel atomically`, - ); - // actually close - this.close(); - } + this.log(`close_if_possible: close this SyncTableChannel atomically`); + // actually close + this.close(); } else { - this.log(`save_and_close_if_possible: NOT closing this SyncTableChannel`); + this.log(`close_if_possible: NOT closing this SyncTableChannel`); } } diff --git a/src/packages/project/sync/sync-doc.ts b/src/packages/project/sync/sync-doc.ts index 7145abb353..0bc510bfe5 100644 --- a/src/packages/project/sync/sync-doc.ts +++ b/src/packages/project/sync/sync-doc.ts @@ -163,6 +163,11 @@ export function get_syncdoc(path: string): SyncDoc | undefined { return syncDocs.get(path); } +export function getSyncDocFromSyncTable(synctable: SyncTable) { + const { opts } = get_type_and_opts(synctable); + return get_syncdoc(opts.path); +} + async function init_syncdoc_async( client: Client, synctable: SyncTable, diff --git a/src/packages/util/smc-version.js b/src/packages/util/smc-version.js index b19e97d8bb..79c35cf353 100644 --- a/src/packages/util/smc-version.js +++ b/src/packages/util/smc-version.js @@ -1,2 +1,2 @@ /* autogenerated by the update_version script */ -exports.version=1707629396; +exports.version=1709012686;