diff --git a/src/packages/backend/files/path-to-files.ts b/src/packages/backend/files/path-to-files.ts new file mode 100644 index 00000000000..fdb344b0f3e --- /dev/null +++ b/src/packages/backend/files/path-to-files.ts @@ -0,0 +1,17 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details + */ + +// This is used to find files on the share server (public_paths) in "next" +// and also in the hub, for deleting shared files of projects + +import { join } from "node:path"; + +import { projects } from "@cocalc/backend/data"; + +// Given a project_id/path, return the directory on the file system where +// that path should be located. +export function pathToFiles(project_id: string, path: string): string { + return join(projects.replace("[project_id]", project_id), path); +} diff --git a/src/packages/database/postgres/delete-projects.ts b/src/packages/database/postgres/delete-projects.ts index bda4b2c7c2d..701e831839a 100644 --- a/src/packages/database/postgres/delete-projects.ts +++ b/src/packages/database/postgres/delete-projects.ts @@ -7,6 +7,10 @@ Code related to permanently deleting projects. */ +import { promises as fs } from "node:fs"; +import { join } from "node:path"; + +import { pathToFiles } from "@cocalc/backend/files/path-to-files"; import getLogger from "@cocalc/backend/logger"; import { newCounter } from "@cocalc/backend/metrics"; import getPool from "@cocalc/database/pool"; @@ -105,7 +109,7 @@ SELECT project_id FROM projects WHERE deleted = true AND users IS NULL - AND state ->> 'state' != 'deleted' + AND coalesce(state ->> 'state', '') != 'deleted' ORDER BY created ASC LIMIT 1000 `; @@ -169,10 +173,12 @@ export async function cleanup_old_projects_data( if (on_prem) { L2(`delete all project files`); - // TODO: this only works on-prem, and requires the project files to be mounted + await deleteProjectFiles(L2, project_id); L2(`deleting all shared files`); - // TODO: do it directly like above, and also get rid of all those shares in the database + // this is something like /shared/projects/${project_id} + const shared_path = pathToFiles(project_id, ""); + await fs.rm(shared_path, { recursive: true, force: true }); // for now, on-prem only as well. This gets rid of all sorts of data in tables specific to the given project. delRows += await delete_associated_project_data(L2, project_id); @@ -261,3 +267,24 @@ async function delete_associated_project_data( return total; } + +async function deleteProjectFiles(L2, project_id: string) { + // TODO: this only works on-prem, and requires the project files to be mounted + const projects_root = process.env["MOUNTED_PROJECTS_ROOT"]; + if (!projects_root) return; + const project_dir = join(projects_root, project_id); + try { + await fs.access(project_dir, fs.constants.F_OK | fs.constants.R_OK); + const stats = await fs.lstat(project_dir); + if (stats.isDirectory()) { + L2(`deleting all files in ${project_dir}`); + await fs.rm(project_dir, { recursive: true, force: true }); + } else { + L2(`is not a directory: ${project_dir}`); + } + } catch (err) { + L2( + `not deleting project files: either it does not exist or is not accessible`, + ); + } +} diff --git a/src/packages/next/lib/share/get-contents.ts b/src/packages/next/lib/share/get-contents.ts index 02d60c23d8a..9f19ca99641 100644 --- a/src/packages/next/lib/share/get-contents.ts +++ b/src/packages/next/lib/share/get-contents.ts @@ -3,14 +3,16 @@ * License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details */ -import pathToFiles from "./path-to-files"; import { promises as fs } from "fs"; -import { join } from "path"; import { sortBy } from "lodash"; +import { join } from "path"; + +import { pathToFiles } from "@cocalc/backend/files/path-to-files"; import { hasSpecialViewer } from "@cocalc/frontend/file-extensions"; import { getExtension } from "./util"; const MB: number = 1000000; + const LIMITS = { listing: 10000, // directory listing is truncated after this many files ipynb: 15 * MB, @@ -18,7 +20,7 @@ const LIMITS = { whiteboard: 5 * MB, slides: 5 * MB, other: 2 * MB, -}; +} as const; export interface FileInfo { name: string; @@ -40,7 +42,7 @@ export interface PathContents { export default async function getContents( project_id: string, - path: string + path: string, ): Promise { const fsPath = pathToFiles(project_id, path); const obj: PathContents = {}; @@ -72,7 +74,7 @@ export default async function getContents( } async function getDirectoryListing( - path: string + path: string, ): Promise<{ listing: FileInfo[]; truncated?: string }> { const listing: FileInfo[] = []; let truncated: string | undefined = undefined; diff --git a/src/packages/next/lib/share/path-to-files.ts b/src/packages/next/lib/share/path-to-files.ts index 1bb0a63449a..ef741bd2a16 100644 --- a/src/packages/next/lib/share/path-to-files.ts +++ b/src/packages/next/lib/share/path-to-files.ts @@ -3,24 +3,17 @@ * License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details */ -import { join } from "path"; +import { pathToFiles } from "@cocalc/backend/files/path-to-files"; import getPool from "@cocalc/database/pool"; -import { projects } from "@cocalc/backend/data"; - -// Given a project_id/path, return the directory on the file system where -// that path should be located. -export default function pathToFiles(project_id: string, path: string): string { - return join(projects.replace("[project_id]", project_id), path); -} export async function pathFromID( - id: string + id: string, ): Promise<{ projectPath: string; fsPath: string }> { // 'infinite' since actually result can't change since id determines the path (it's a reverse sha1 hash computation) const pool = getPool("infinite"); const { rows } = await pool.query( "SELECT project_id, path FROM public_paths WHERE id=$1 AND disabled IS NOT TRUE", - [id] + [id], ); if (rows.length == 0) { throw Error(`no such public path: ${id}`); diff --git a/src/packages/next/lib/share/virtual-hosts.ts b/src/packages/next/lib/share/virtual-hosts.ts index 3c1aa69cbcd..95be46785d8 100644 --- a/src/packages/next/lib/share/virtual-hosts.ts +++ b/src/packages/next/lib/share/virtual-hosts.ts @@ -8,12 +8,13 @@ Support for virtual hosts. */ import type { Request, Response } from "express"; + +import basePath from "@cocalc/backend/base-path"; +import { pathToFiles } from "@cocalc/backend/files/path-to-files"; import { getLogger } from "@cocalc/backend/logger"; -import pathToFiles from "./path-to-files"; import isAuthenticated from "./authenticate"; import getVirtualHostInfo from "./get-vhost-info"; import { staticHandler } from "./handle-raw"; -import basePath from "@cocalc/backend/base-path"; const logger = getLogger("virtual-hosts"); @@ -23,7 +24,7 @@ export default function virtualHostsMiddleware() { return async function ( req: Request, res: Response, - next: Function + next: Function, ): Promise { // For debugging in cc-in-cc dev, just manually set host to something // else and comment this out. That's the only way, since dev is otherwise @@ -69,7 +70,7 @@ export default function virtualHostsMiddleware() { logger.debug( "not authenticated -- denying vhost='%s', path='%s'", vhost, - path + path, ); res.status(403).end(); return;