From ad453da8592c7bce273cd78a07ca78ec665b303f Mon Sep 17 00:00:00 2001 From: Laurin Weger <59744144+Laurin-W@users.noreply.github.com> Date: Mon, 20 May 2024 21:41:33 +0200 Subject: [PATCH 1/8] chore: service for actor deletion --- .../packages/auth/services/account.js | 23 ++ src/middleware/packages/backup/index.js | 68 +++++- .../packages/backup/indexTypes.d.ts | 2 +- .../packages/backup/utils/fsRemove.js | 12 ++ .../packages/backup/utils/ftpRemove.js | 25 +++ src/middleware/packages/delete/LICENSE | 201 ++++++++++++++++++ src/middleware/packages/delete/README.md | 5 + src/middleware/packages/delete/index.js | 56 +++++ src/middleware/packages/delete/package.json | 16 ++ .../packages/triplestore/service.js | 4 +- .../triplestore/subservices/dataset.js | 41 +++- src/middleware/tests/config.js | 3 +- website/docs/middleware/backup.md | 33 ++- 13 files changed, 455 insertions(+), 34 deletions(-) create mode 100644 src/middleware/packages/backup/utils/fsRemove.js create mode 100644 src/middleware/packages/backup/utils/ftpRemove.js create mode 100644 src/middleware/packages/delete/LICENSE create mode 100644 src/middleware/packages/delete/README.md create mode 100644 src/middleware/packages/delete/index.js create mode 100644 src/middleware/packages/delete/package.json diff --git a/src/middleware/packages/auth/services/account.js b/src/middleware/packages/auth/services/account.js index ac330c643..f1ed154e8 100644 --- a/src/middleware/packages/auth/services/account.js +++ b/src/middleware/packages/auth/services/account.js @@ -184,6 +184,29 @@ module.exports = { '@id': account['@id'], ...params }); + }, + async deleteByWebId(ctx) { + const { webId } = ctx.params; + const account = await ctx.call('auth.account.findByWebId', { webId }); + + if (account) { + await this._remove(ctx, { id: account['@id'] }); + return true; + } + + return false; + }, + // Remove email and password from an account, set deletedAt timestamp. + async setTombstone(ctx) { + const { webId } = ctx.params; + const account = await ctx.call('auth.account.findByWebId', { webId }); + + return await this._update(ctx, { + '@id': account['@id'], + email: undefined, + hashedPassword: undefined, + deletedAt: new Date().toISOString() + }); } }, methods: { diff --git a/src/middleware/packages/backup/index.js b/src/middleware/packages/backup/index.js index 32d945151..c1e355e68 100644 --- a/src/middleware/packages/backup/index.js +++ b/src/middleware/packages/backup/index.js @@ -1,7 +1,10 @@ const { CronJob } = require('cron'); +const fs = require('fs'); const fsCopy = require('./utils/fsCopy'); const ftpCopy = require('./utils/ftpCopy'); const rsyncCopy = require('./utils/rsyncCopy'); +const ftpRemove = require('./utils/ftpRemove'); +const fsRemove = require('./utils/fsRemove'); /** * @typedef {import('moleculer').Context} Context */ @@ -10,7 +13,7 @@ const BackupService = { name: 'backup', settings: { localServer: { - fusekiBackupsPath: null, + fusekiBase: null, otherDirsPaths: {} }, copyMethod: 'rsync', // rsync, ftp, or fs @@ -34,6 +37,10 @@ const BackupService = { if (cronJob.time) { this.cronJob = new CronJob(cronJob.time, this.actions.backupAll, null, true, cronJob.timeZone); } + + if (!this.settings.localServer.fusekiBase) { + throw new Error('Backup service requires `localServer.fusekiBase` setting to be set to the FUSEKI_BASE path.'); + } }, actions: { async backupAll(ctx) { @@ -41,13 +48,6 @@ const BackupService = { await this.actions.backupOtherDirs({}, { parentCtx: ctx }); }, async backupDatasets(ctx) { - const { fusekiBackupsPath } = this.settings.localServer; - - if (!fusekiBackupsPath) { - this.logger.info('No fusekiBackupsPath defined, skipping backup...'); - return; - } - // Generate a new backup of all datasets const datasets = await ctx.call('triplestore.dataset.list'); for (const dataset of datasets) { @@ -55,7 +55,10 @@ const BackupService = { await ctx.call('triplestore.dataset.backup', { dataset }); } - await this.actions.copyToRemoteServer({ path: fusekiBackupsPath, subDir: 'datasets' }, { parentCtx: ctx }); + await this.actions.copyToRemoteServer( + { path: path.join(this.settings.localServer.fusekiBase, 'backups'), subDir: 'datasets' }, + { parentCtx: ctx } + ); }, async backupOtherDirs(ctx) { const { otherDirsPaths } = this.settings.localServer; @@ -96,6 +99,53 @@ const BackupService = { default: throw new Error(`Unknown copy method: ${copyMethod}`); } + }, + deleteDataset: { + params: { + dataset: { type: 'string' }, + iKnowWhatImDoing: { type: 'boolean' } + }, + async handle(ctx) { + const { dataset, iKnowWhatImDoing } = ctx.params; + const { + copyMethod, + remoteServer, + localServer: { fusekiBase } + } = this.settings; + if (!iKnowWhatImDoing) { + throw new Error( + 'Please confirm that you know what you are doing and set the `iKnowWhatImDoing` parameter to `true`.' + ); + } + + // File format: _ + const backupsPattern = RegExp(`^${dataset}_.{10}_.{8}\\.nq\\.gz$`); + const deleteFilenames = await fs.promises + .readdir(path.join(fusekiBase, 'backups')) + .then(files => files.filter(file => backupsPattern.test(file))); + + // Delete all backups locally. + await Promise.all(deleteFilenames.map(file => path.join('./backups', file)).map(file => fs.promises.rm(file))); + + // Delete backups from remote. + switch (copyMethod) { + case 'rsync': + // This will sync deletions too. + await rsyncCopy(path.join(fusekiBase, 'backups'), 'datasets', remoteServer); + break; + + case 'ftp': + await ftpRemove(deleteFilenames, remoteServer); + break; + + case 'fs': + await fsRemove(deleteFilenames, remoteServer); + break; + + default: + throw new Error(`Unknown copy method: ${copyMethod}`); + } + } } } }; diff --git a/src/middleware/packages/backup/indexTypes.d.ts b/src/middleware/packages/backup/indexTypes.d.ts index 81e1f94f8..200316275 100644 --- a/src/middleware/packages/backup/indexTypes.d.ts +++ b/src/middleware/packages/backup/indexTypes.d.ts @@ -2,7 +2,7 @@ import { Context, ServiceSchema, CallingOptions } from 'moleculer'; import { CronJob } from 'cron'; interface LocalServerSettings { - fusekiBackupsPath: string | null; + fusekiBase: string | null; otherDirsPaths: Record; } diff --git a/src/middleware/packages/backup/utils/fsRemove.js b/src/middleware/packages/backup/utils/fsRemove.js new file mode 100644 index 000000000..04ce06579 --- /dev/null +++ b/src/middleware/packages/backup/utils/fsRemove.js @@ -0,0 +1,12 @@ +const fs = require('fs'); +const { join: pathJoin } = require('path'); + +const fsRemove = async (removeFiles, remoteServer) => { + await Promise.all( + removeFiles + .map(file => pathJoin(remoteServer.path, subDir, file)) + .map(file => fs.promises.rm(file, { force: true })) + ); +}; + +module.exports = fsRemove; diff --git a/src/middleware/packages/backup/utils/ftpRemove.js b/src/middleware/packages/backup/utils/ftpRemove.js new file mode 100644 index 000000000..4bb3bfedd --- /dev/null +++ b/src/middleware/packages/backup/utils/ftpRemove.js @@ -0,0 +1,25 @@ +const fs = require('fs'); +const Client = require('ssh2-sftp-client'); +const { join: pathJoin } = require('path'); + +const ftpRemove = (removeFiles, remoteServer) => { + return new Promise((resolve, reject) => { + const sftp = new Client(); + sftp + .connect({ + host: remoteServer.host, + port: remoteServer.port, + username: remoteServer.user, + password: remoteServer.password + }) + .then(async () => { + for (const filename of removeFiles) { + await sftp.delete(pathJoin(remoteServer.path, filename), true); + } + resolve(); + }) + .catch(e => reject(e)); + }); +}; + +module.exports = ftpRemove; diff --git a/src/middleware/packages/delete/LICENSE b/src/middleware/packages/delete/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/src/middleware/packages/delete/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/middleware/packages/delete/README.md b/src/middleware/packages/delete/README.md new file mode 100644 index 000000000..24fca6c57 --- /dev/null +++ b/src/middleware/packages/delete/README.md @@ -0,0 +1,5 @@ +# @semapps/delete + +Service to delete dataset and uploaded files. + +[Documentation](https://semapps.org/docs/middleware/delete) diff --git a/src/middleware/packages/delete/index.js b/src/middleware/packages/delete/index.js new file mode 100644 index 000000000..06519e547 --- /dev/null +++ b/src/middleware/packages/delete/index.js @@ -0,0 +1,56 @@ +const { CronJob } = require('cron'); +const { getDatasetFromUri } = require('@semapps/ldp'); +const path = require('node:path'); +const fs = require('fs'); + +/** @type {import('moleculer').ServiceSchema} */ +const DeleteService = { + name: 'delete', + settings: { + settingsDataset: 'settings' + }, + actions: { + deleteActor: { + params: { + webId: { type: 'string' }, + iKnowWhatImDoing: { type: 'boolean' } + }, + async handler(ctx) { + const { webId, iKnowWhatImDoing } = ctx.params; + if (!iKnowWhatImDoing) { + throw new Error( + 'Please confirm that you know what you are doing and set the `iKnowWhatImDoing` parameter to `true`.' + ); + } + const dataset = getDatasetFromUri(webId); + + // Delete keys (this will only take effect, if the key store is still in legacy state). + const deleteKeyPromise = ctx.call('signature.keypair.delete', { actorUri: webId }); + + // Delete dataset. + const delDatasetPromise = ctx.call('dataset.delete', { dataset, iKnowWhatImDoing }); + + // Delete account information settings data. + const deleteAccountPromise = ctx.call('auth.account.setTombstone', { webid }); + + // Delete uploads. + const uploadsPath = path.join('./uploads/', dataset); + const delUploadsPromise = fs.promises.rm(uploadsPath, { recursive: true, force: true }); + + // Delete backups. + let delBackupsPromise; + if (ctx.broker.registry.hasService('backup')) { + delBackupsPromise = ctx.call('backup.deleteDataset', { iKnowWhatImDoing, dataset }); + } + + await delDatasetPromise; + await deleteKeyPromise; + await deleteAccountPromise; + await delUploadsPromise; + await delBackupsPromise; + } + } + } +}; + +module.exports = { DeleteService }; diff --git a/src/middleware/packages/delete/package.json b/src/middleware/packages/delete/package.json new file mode 100644 index 000000000..4e1ac050d --- /dev/null +++ b/src/middleware/packages/delete/package.json @@ -0,0 +1,16 @@ +{ + "name": "@semapps/delete", + "version": "0.7.0", + "description": "Service to delete dataset and uploaded files", + "license": "Apache-2.0", + "author": "Virtual Assembly", + "dependencies": { + "moleculer": "^0.14.31" + }, + "publishConfig": { + "access": "public" + }, + "engines": { + "node": ">=14" + } +} diff --git a/src/middleware/packages/triplestore/service.js b/src/middleware/packages/triplestore/service.js index 65a7d29d9..e08a09b0a 100644 --- a/src/middleware/packages/triplestore/service.js +++ b/src/middleware/packages/triplestore/service.js @@ -19,12 +19,13 @@ const TripleStoreService = { user: null, password: null, mainDataset: null, + fusekiBase: null, // Sub-services customization dataset: {} }, dependencies: ['jsonld'], async created() { - const { url, user, password, dataset } = this.settings; + const { url, user, password, dataset, fusekiBase } = this.settings; this.subservices = {}; if (dataset !== false) { @@ -33,6 +34,7 @@ const TripleStoreService = { url, user, password, + fusekiBase, ...dataset } }); diff --git a/src/middleware/packages/triplestore/subservices/dataset.js b/src/middleware/packages/triplestore/subservices/dataset.js index 3f3ab3685..8ec5348be 100644 --- a/src/middleware/packages/triplestore/subservices/dataset.js +++ b/src/middleware/packages/triplestore/subservices/dataset.js @@ -1,17 +1,19 @@ const fetch = require('node-fetch'); -const fsPromises = require('fs').promises; +const fs = require('fs'); const path = require('path'); const urlJoin = require('url-join'); const format = require('string-template'); const delay = t => new Promise(resolve => setTimeout(resolve, t)); +/** @type {import('moleculer').ServiceSchema} */ const DatasetService = { name: 'triplestore.dataset', settings: { url: null, user: null, - password: null + password: null, + fusekiBase: null }, started() { this.headers = { @@ -44,7 +46,7 @@ const DatasetService = { throw new Error(`Error when creating dataset ${dataset}. Its name cannot end with Acl or Mirror`); const templateFilePath = path.join(__dirname, '../templates', secure ? 'secure-dataset.ttl' : 'dataset.ttl'); - const template = await fsPromises.readFile(templateFilePath, 'utf8'); + const template = await fs.promises.readFile(templateFilePath, 'utf8'); const assembler = format(template, { dataset: dataset }); response = await fetch(urlJoin(this.settings.url, '$/datasets'), { method: 'POST', @@ -104,6 +106,39 @@ const DatasetService = { } } while (!task || !task.finished); } + }, + delete: { + params: { + dataset: { type: 'string' }, + iKnowWhatImDoing: { type: 'boolean' } + }, + async handler(ctx) { + const { dataset, iKnowWhatImDoing } = ctx.params; + if (!iKnowWhatImDoing) { + throw new Error('Please confirm that you know what you are doing by setting `iKnowWhatImDoing` to `true`.'); + } + + const response = await fetch(urlJoin(this.settings.url, '$/datasets', dataset), { + method: 'DELETE', + headers: this.headers + }); + const { taskId } = await response.json(); + await this.actions.waitForTaskCompletion({ taskId }, { parentCtx: ctx }); + + // If this is a secure dataset, we might need to delete stuff manually. + if (this.settings.fusekiBase) { + const dbDir = path.join(this.settings.fusekiBase, 'databases', dataset); + const dbAclDir = path.join(this.settings.fusekiBase, 'databases', `${dataset}Acl`); + const confFile = path.join(this.settings.fusekiBase, 'configuration', `${dataset}.ttl`); + + // Delete all, if present. + await Promise.all([ + fs.promises.rm(dbDir, { recursive: true, force: true }), + fs.promises.rm(dbAclDir, { recursive: true, force: true }), + fs.promises.rm(confFile, { force: true }) + ]); + } + } } }; diff --git a/src/middleware/tests/config.js b/src/middleware/tests/config.js index 9bd0f7d66..cd330e4d4 100644 --- a/src/middleware/tests/config.js +++ b/src/middleware/tests/config.js @@ -9,5 +9,6 @@ module.exports = { SETTINGS_DATASET: process.env.SEMAPPS_SETTINGS_DATASET, JENA_USER: process.env.SEMAPPS_JENA_USER, JENA_PASSWORD: process.env.SEMAPPS_JENA_PASSWORD, - ACTIVATE_CACHE: process.env.SEMAPPS_ACTIVATE_CACHE === 'true' + ACTIVATE_CACHE: process.env.SEMAPPS_ACTIVATE_CACHE === 'true', + FUSEKI_BASE: process.env.FUSEKI_BASE }; diff --git a/website/docs/middleware/backup.md b/website/docs/middleware/backup.md index 133a8ec38..9c1ad4253 100644 --- a/website/docs/middleware/backup.md +++ b/website/docs/middleware/backup.md @@ -4,19 +4,16 @@ title: Backup This service allows you to backup the triples in a given dataset, as well as the uploaded files. - ## Features - Backup Fuseki datasets and uploaded files - Choose copy method: Rsync, FTP or filesystem (copy to another directory) - Setup a cron to automatically launch the rsync operation - ## Dependencies - [TripleStoreService](triplestore) - ## Install ```bash @@ -35,7 +32,6 @@ You will also need to add the remote server domain as a known host, otherwise ss ssh-keyscan REMOTE_SERVER_DOMAIN_NAME >> ~/.ssh/known_hosts ``` - ## Usage ```js @@ -46,7 +42,7 @@ module.exports = { mixins: [BackupService], settings: { localServer: { - fusekiBackupsPath: '/absolute/path/to/fuseki/backups', + fusekiBase: '/absolute/path/to/fuseki-base/', otherDirsPaths: { uploads: path.resolve(__dirname, '../uploads') } @@ -57,8 +53,7 @@ module.exports = { user: 'user', // Required for rsync and ftp password: 'password', // Required for rsync and ftp host: 'remote.server.com', // Required for rsync and ftp - port: null, // Required for ftp - + port: null // Required for ftp }, // Required only if you want to do automatic backups cronJob: { @@ -69,16 +64,15 @@ module.exports = { }; ``` - ## Service settings -| Property | Type | Default | Description | -|----------------|------------|---------|------------------------------------------------------------------------------| -| `localServer` | `[Object]` | | Absolute path to the Fuseki backups and other directories you want to backup | -| `copyMethod` | `[String]` | "rsync" | Copy method ("rsync", "ftp" or "fs") | -| `remoteServer` | `[Object]` | | Information to connect to the remote server (see above) | -| `cronJob` | `[Object]` | | Information for the automatic backups (see above) | - +| Property | Type | Default | Description | +| ---------------------------- | ------------------------ | ------- | ---------------------------------------------------------------------------- | +| `localServer.fusekiBase` | `[String]` | | Absolute path to the Fuseki backups and other directories you want to backup | +| `localServer.otherDirsPaths` | `Record` | | Other directories to back up with the keys as the backup dir names. | +| `copyMethod` | `[String]` | "rsync" | Copy method ("rsync", "ftp" or "fs") | +| `remoteServer` | `[Object]` | | Information to connect to the remote server (see above) | +| `cronJob` | `[Object]` | | Information for the automatic backups (see above) | ## Actions @@ -101,7 +95,8 @@ Copy the other directories defined in the settings with the remote server. Copy the data in the local server to the remote server. ##### Parameters -| Property | Type | Default | Description | -|----------| ---- |--------------------|---------------------------------------------------------| -| `path` | `String` | **required** | Absolute path to be synchronized with the remote server | -| `subDir` | `String` | | Sub-directory in the remote server | + +| Property | Type | Default | Description | +| -------- | -------- | ------------ | ------------------------------------------------------- | +| `path` | `String` | **required** | Absolute path to be synchronized with the remote server | +| `subDir` | `String` | | Sub-directory in the remote server | From 2fbb3d9654b006c855201010f8174fd7908b3a22 Mon Sep 17 00:00:00 2001 From: Laurin Weger <59744144+Laurin-W@users.noreply.github.com> Date: Tue, 21 May 2024 11:28:01 +0200 Subject: [PATCH 2/8] add `.management/actor` route --- src/middleware/packages/delete/index.js | 32 +++++++++++++++++--- src/middleware/packages/middlewares/index.js | 6 ++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/middleware/packages/delete/index.js b/src/middleware/packages/delete/index.js index 06519e547..1ec43f622 100644 --- a/src/middleware/packages/delete/index.js +++ b/src/middleware/packages/delete/index.js @@ -2,6 +2,7 @@ const { CronJob } = require('cron'); const { getDatasetFromUri } = require('@semapps/ldp'); const path = require('node:path'); const fs = require('fs'); +const { throw403 } = require('@semapps/middlewares'); /** @type {import('moleculer').ServiceSchema} */ const DeleteService = { @@ -9,29 +10,50 @@ const DeleteService = { settings: { settingsDataset: 'settings' }, + async started(ctx) { + ctx.call('api.addRoute', { + route: { + name: 'management', + path: '/.management/actor/:actorSlug', + aliases: { + 'DELETE /': 'delete.deleteActor' + } + } + }); + }, actions: { deleteActor: { params: { - webId: { type: 'string' }, + actorSlug: { type: 'string' }, iKnowWhatImDoing: { type: 'boolean' } }, async handler(ctx) { - const { webId, iKnowWhatImDoing } = ctx.params; + const { actorSlug: dataset, iKnowWhatImDoing } = ctx.params; + const webId = ctx.meta.webId; if (!iKnowWhatImDoing) { throw new Error( 'Please confirm that you know what you are doing and set the `iKnowWhatImDoing` parameter to `true`.' ); } - const dataset = getDatasetFromUri(webId); + + if (getDatasetFromUri(webId) !== dataset && webId !== 'system') { + throw403('You are not allowed to delete this actor.'); + } + + // Validate that the actor exists. + const actorUri = await ctx.call('auth.account.findByUsername', { username: dataset }); + if (!actorUri) { + throw404('Actor not found.'); + } // Delete keys (this will only take effect, if the key store is still in legacy state). - const deleteKeyPromise = ctx.call('signature.keypair.delete', { actorUri: webId }); + const deleteKeyPromise = ctx.call('signature.keypair.delete', { actorUri }); // Delete dataset. const delDatasetPromise = ctx.call('dataset.delete', { dataset, iKnowWhatImDoing }); // Delete account information settings data. - const deleteAccountPromise = ctx.call('auth.account.setTombstone', { webid }); + const deleteAccountPromise = ctx.call('auth.account.setTombstone', { actorUri }); // Delete uploads. const uploadsPath = path.join('./uploads/', dataset); diff --git a/src/middleware/packages/middlewares/index.js b/src/middleware/packages/middlewares/index.js index dc0582786..2078e1741 100644 --- a/src/middleware/packages/middlewares/index.js +++ b/src/middleware/packages/middlewares/index.js @@ -47,6 +47,11 @@ const throw403 = msg => { throw new MoleculerError('Forbidden', 403, 'ACCESS_DENIED', { status: 'Forbidden', text: msg }); }; +/** @type {(msg: string) => never} */ +const throw404 = msg => { + throw new MoleculerError('Forbidden', 404, 'NOT_FOUND', { status: 'Not found', text: msg }); +}; + const throw500 = msg => { throw new MoleculerError(msg, 500, 'INTERNAL_SERVER_ERROR', { status: 'Server Error', text: msg }); }; @@ -202,5 +207,6 @@ module.exports = { saveDatasetMeta, throw400, throw403, + throw404, throw500 }; From 434b7775997ffa4cc1276e430c71e8bc366e2084 Mon Sep 17 00:00:00 2001 From: Laurin Weger <59744144+Laurin-W@users.noreply.github.com> Date: Tue, 21 May 2024 12:13:15 +0200 Subject: [PATCH 3/8] move delete service to apods management service, minor things --- src/middleware/packages/core/service.js | 9 +- src/middleware/packages/delete/LICENSE | 201 ------------------ src/middleware/packages/delete/README.md | 5 - src/middleware/packages/delete/index.js | 78 ------- src/middleware/packages/delete/package.json | 16 -- .../triplestore/subservices/dataset.js | 16 +- 6 files changed, 18 insertions(+), 307 deletions(-) delete mode 100644 src/middleware/packages/delete/LICENSE delete mode 100644 src/middleware/packages/delete/README.md delete mode 100644 src/middleware/packages/delete/index.js delete mode 100644 src/middleware/packages/delete/package.json diff --git a/src/middleware/packages/core/service.js b/src/middleware/packages/core/service.js index 5acefe2f5..a47c59f17 100644 --- a/src/middleware/packages/core/service.js +++ b/src/middleware/packages/core/service.js @@ -35,7 +35,8 @@ const CoreService = { url: undefined, user: undefined, password: undefined, - mainDataset: undefined + mainDataset: undefined, + fusekiBase: undefined }, // Optional containers: undefined, @@ -173,11 +174,7 @@ const CoreService = { this.broker.createService(TripleStoreService, { settings: { - url: triplestore.url, - user: triplestore.user, - password: triplestore.password, - mainDataset: triplestore.mainDataset, - ...this.settings.triplestore + ...triplestore }, async started() { if (triplestore.mainDataset) { diff --git a/src/middleware/packages/delete/LICENSE b/src/middleware/packages/delete/LICENSE deleted file mode 100644 index 261eeb9e9..000000000 --- a/src/middleware/packages/delete/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/src/middleware/packages/delete/README.md b/src/middleware/packages/delete/README.md deleted file mode 100644 index 24fca6c57..000000000 --- a/src/middleware/packages/delete/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# @semapps/delete - -Service to delete dataset and uploaded files. - -[Documentation](https://semapps.org/docs/middleware/delete) diff --git a/src/middleware/packages/delete/index.js b/src/middleware/packages/delete/index.js deleted file mode 100644 index 1ec43f622..000000000 --- a/src/middleware/packages/delete/index.js +++ /dev/null @@ -1,78 +0,0 @@ -const { CronJob } = require('cron'); -const { getDatasetFromUri } = require('@semapps/ldp'); -const path = require('node:path'); -const fs = require('fs'); -const { throw403 } = require('@semapps/middlewares'); - -/** @type {import('moleculer').ServiceSchema} */ -const DeleteService = { - name: 'delete', - settings: { - settingsDataset: 'settings' - }, - async started(ctx) { - ctx.call('api.addRoute', { - route: { - name: 'management', - path: '/.management/actor/:actorSlug', - aliases: { - 'DELETE /': 'delete.deleteActor' - } - } - }); - }, - actions: { - deleteActor: { - params: { - actorSlug: { type: 'string' }, - iKnowWhatImDoing: { type: 'boolean' } - }, - async handler(ctx) { - const { actorSlug: dataset, iKnowWhatImDoing } = ctx.params; - const webId = ctx.meta.webId; - if (!iKnowWhatImDoing) { - throw new Error( - 'Please confirm that you know what you are doing and set the `iKnowWhatImDoing` parameter to `true`.' - ); - } - - if (getDatasetFromUri(webId) !== dataset && webId !== 'system') { - throw403('You are not allowed to delete this actor.'); - } - - // Validate that the actor exists. - const actorUri = await ctx.call('auth.account.findByUsername', { username: dataset }); - if (!actorUri) { - throw404('Actor not found.'); - } - - // Delete keys (this will only take effect, if the key store is still in legacy state). - const deleteKeyPromise = ctx.call('signature.keypair.delete', { actorUri }); - - // Delete dataset. - const delDatasetPromise = ctx.call('dataset.delete', { dataset, iKnowWhatImDoing }); - - // Delete account information settings data. - const deleteAccountPromise = ctx.call('auth.account.setTombstone', { actorUri }); - - // Delete uploads. - const uploadsPath = path.join('./uploads/', dataset); - const delUploadsPromise = fs.promises.rm(uploadsPath, { recursive: true, force: true }); - - // Delete backups. - let delBackupsPromise; - if (ctx.broker.registry.hasService('backup')) { - delBackupsPromise = ctx.call('backup.deleteDataset', { iKnowWhatImDoing, dataset }); - } - - await delDatasetPromise; - await deleteKeyPromise; - await deleteAccountPromise; - await delUploadsPromise; - await delBackupsPromise; - } - } - } -}; - -module.exports = { DeleteService }; diff --git a/src/middleware/packages/delete/package.json b/src/middleware/packages/delete/package.json deleted file mode 100644 index 4e1ac050d..000000000 --- a/src/middleware/packages/delete/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "@semapps/delete", - "version": "0.7.0", - "description": "Service to delete dataset and uploaded files", - "license": "Apache-2.0", - "author": "Virtual Assembly", - "dependencies": { - "moleculer": "^0.14.31" - }, - "publishConfig": { - "access": "public" - }, - "engines": { - "node": ">=14" - } -} diff --git a/src/middleware/packages/triplestore/subservices/dataset.js b/src/middleware/packages/triplestore/subservices/dataset.js index 8ec5348be..3e801187f 100644 --- a/src/middleware/packages/triplestore/subservices/dataset.js +++ b/src/middleware/packages/triplestore/subservices/dataset.js @@ -81,6 +81,15 @@ const DatasetService = { } return []; }, + async isSecure(ctx) { + const { dataset } = ctx.params; + // Check if http://semapps.org/webacl graph exists + return await ctx.call('triplestore.query', { + query: `ASK WHERE { GRAPH { ?s ?p ?o } }`, + dataset, + webId: 'system' + }); + }, async waitForCreation(ctx) { const { dataset } = ctx.params; let datasetExist; @@ -126,7 +135,12 @@ const DatasetService = { await this.actions.waitForTaskCompletion({ taskId }, { parentCtx: ctx }); // If this is a secure dataset, we might need to delete stuff manually. - if (this.settings.fusekiBase) { + if (await this.actions.isSecure({ dataset })) { + if (!this.settings.fusekiBase) + throw new Error( + 'Please provide the fusekiBase dir setting to the triplestore service, to delete a secure dataset.' + ); + const dbDir = path.join(this.settings.fusekiBase, 'databases', dataset); const dbAclDir = path.join(this.settings.fusekiBase, 'databases', `${dataset}Acl`); const confFile = path.join(this.settings.fusekiBase, 'configuration', `${dataset}.ttl`); From 40f67d4cb16634bebaa1e706f5c725a017c08294 Mon Sep 17 00:00:00 2001 From: Laurin Weger <59744144+Laurin-W@users.noreply.github.com> Date: Tue, 21 May 2024 19:48:29 +0200 Subject: [PATCH 4/8] running unit tests --- .../triplestore/subservices/dataset.js | 62 ++++++++++--------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/src/middleware/packages/triplestore/subservices/dataset.js b/src/middleware/packages/triplestore/subservices/dataset.js index 3e801187f..b9053ac77 100644 --- a/src/middleware/packages/triplestore/subservices/dataset.js +++ b/src/middleware/packages/triplestore/subservices/dataset.js @@ -114,43 +114,45 @@ const DatasetService = { task = await response.json(); } } while (!task || !task.finished); - } - }, - delete: { - params: { - dataset: { type: 'string' }, - iKnowWhatImDoing: { type: 'boolean' } }, - async handler(ctx) { - const { dataset, iKnowWhatImDoing } = ctx.params; - if (!iKnowWhatImDoing) { - throw new Error('Please confirm that you know what you are doing by setting `iKnowWhatImDoing` to `true`.'); - } - - const response = await fetch(urlJoin(this.settings.url, '$/datasets', dataset), { - method: 'DELETE', - headers: this.headers - }); - const { taskId } = await response.json(); - await this.actions.waitForTaskCompletion({ taskId }, { parentCtx: ctx }); + delete: { + params: { + dataset: { type: 'string' }, + iKnowWhatImDoing: { type: 'boolean' } + }, + async handler(ctx) { + const { dataset, iKnowWhatImDoing } = ctx.params; + if (!iKnowWhatImDoing) { + throw new Error('Please confirm that you know what you are doing by setting `iKnowWhatImDoing` to `true`.'); + } + const isSecure = await this.actions.isSecure({ dataset }); - // If this is a secure dataset, we might need to delete stuff manually. - if (await this.actions.isSecure({ dataset })) { - if (!this.settings.fusekiBase) + if (isSecure && !this.settings.fusekiBase) throw new Error( 'Please provide the fusekiBase dir setting to the triplestore service, to delete a secure dataset.' ); - const dbDir = path.join(this.settings.fusekiBase, 'databases', dataset); - const dbAclDir = path.join(this.settings.fusekiBase, 'databases', `${dataset}Acl`); - const confFile = path.join(this.settings.fusekiBase, 'configuration', `${dataset}.ttl`); + const response = await fetch(urlJoin(this.settings.url, '$/datasets', dataset), { + method: 'DELETE', + headers: this.headers + }); + if (!response.ok) { + throw new Error(`Failed to delete dataset ${dataset}: ${response.statusText}`); + } - // Delete all, if present. - await Promise.all([ - fs.promises.rm(dbDir, { recursive: true, force: true }), - fs.promises.rm(dbAclDir, { recursive: true, force: true }), - fs.promises.rm(confFile, { force: true }) - ]); + // If this is a secure dataset, we might need to delete stuff manually. + if (isSecure) { + const dbDir = path.join(this.settings.fusekiBase, 'databases', dataset); + const dbAclDir = path.join(this.settings.fusekiBase, 'databases', `${dataset}Acl`); + const confFile = path.join(this.settings.fusekiBase, 'configuration', `${dataset}.ttl`); + + // Delete all, if present. + await Promise.all([ + fs.promises.rm(dbDir, { recursive: true, force: true }), + fs.promises.rm(dbAclDir, { recursive: true, force: true }), + fs.promises.rm(confFile, { force: true }) + ]); + } } } } From 157502e20813108b7edade536808aea9b4f380b7 Mon Sep 17 00:00:00 2001 From: Laurin Weger <59744144+Laurin-W@users.noreply.github.com> Date: Thu, 23 May 2024 12:47:08 +0200 Subject: [PATCH 5/8] backup service delete method fixes --- .../packages/auth/services/account.js | 9 +++++-- src/middleware/packages/backup/index.js | 26 ++++++++++++------- .../packages/backup/utils/fsRemove.js | 2 +- .../packages/backup/utils/ftpRemove.js | 1 - .../packages/backup/utils/rsyncCopy.js | 4 ++- .../triplestore/subservices/dataset.js | 4 ++- 6 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/middleware/packages/auth/services/account.js b/src/middleware/packages/auth/services/account.js index f1ed154e8..6bdff69b6 100644 --- a/src/middleware/packages/auth/services/account.js +++ b/src/middleware/packages/auth/services/account.js @@ -202,9 +202,14 @@ module.exports = { const account = await ctx.call('auth.account.findByWebId', { webId }); return await this._update(ctx, { + // Set all values to undefined... + ...Object.fromEntries(Object.keys(account).map(key => [key, null])), '@id': account['@id'], - email: undefined, - hashedPassword: undefined, + // ...except for + webId: account.webId, + username: account.username, + podUri: account.podUri, + // And add a deletedAt date. deletedAt: new Date().toISOString() }); } diff --git a/src/middleware/packages/backup/index.js b/src/middleware/packages/backup/index.js index c1e355e68..2533d9d37 100644 --- a/src/middleware/packages/backup/index.js +++ b/src/middleware/packages/backup/index.js @@ -1,5 +1,6 @@ const { CronJob } = require('cron'); const fs = require('fs'); +const pathJoin = require('path').join; const fsCopy = require('./utils/fsCopy'); const ftpCopy = require('./utils/ftpCopy'); const rsyncCopy = require('./utils/rsyncCopy'); @@ -32,13 +33,16 @@ const BackupService = { }, dependencies: ['triplestore'], started() { - const { cronJob } = this.settings; + const { + cronJob, + localServer: { fusekiBase } + } = this.settings; if (cronJob.time) { this.cronJob = new CronJob(cronJob.time, this.actions.backupAll, null, true, cronJob.timeZone); } - if (!this.settings.localServer.fusekiBase) { + if (!fusekiBase) { throw new Error('Backup service requires `localServer.fusekiBase` setting to be set to the FUSEKI_BASE path.'); } }, @@ -56,7 +60,7 @@ const BackupService = { } await this.actions.copyToRemoteServer( - { path: path.join(this.settings.localServer.fusekiBase, 'backups'), subDir: 'datasets' }, + { path: pathJoin(this.settings.localServer.fusekiBase, 'backups'), subDir: 'datasets' }, { parentCtx: ctx } ); }, @@ -105,7 +109,7 @@ const BackupService = { dataset: { type: 'string' }, iKnowWhatImDoing: { type: 'boolean' } }, - async handle(ctx) { + async handler(ctx) { const { dataset, iKnowWhatImDoing } = ctx.params; const { copyMethod, @@ -121,17 +125,19 @@ const BackupService = { // File format: _ const backupsPattern = RegExp(`^${dataset}_.{10}_.{8}\\.nq\\.gz$`); const deleteFilenames = await fs.promises - .readdir(path.join(fusekiBase, 'backups')) + .readdir(pathJoin(fusekiBase, 'backups')) .then(files => files.filter(file => backupsPattern.test(file))); // Delete all backups locally. - await Promise.all(deleteFilenames.map(file => path.join('./backups', file)).map(file => fs.promises.rm(file))); + await Promise.all( + deleteFilenames.map(file => pathJoin(fusekiBase, 'backups', file)).map(file => fs.promises.rm(file)) + ); - // Delete backups from remote. + // Delete backups from remote.fusekiBase switch (copyMethod) { case 'rsync': - // This will sync deletions too. - await rsyncCopy(path.join(fusekiBase, 'backups'), 'datasets', remoteServer); + // The last param sets the --deletion argument, to sync deletions too. + await rsyncCopy(pathJoin(fusekiBase, 'backups'), 'datasets', remoteServer, true); break; case 'ftp': @@ -139,7 +145,7 @@ const BackupService = { break; case 'fs': - await fsRemove(deleteFilenames, remoteServer); + await fsRemove(deleteFilenames, 'datasets', remoteServer); break; default: diff --git a/src/middleware/packages/backup/utils/fsRemove.js b/src/middleware/packages/backup/utils/fsRemove.js index 04ce06579..7a4315d0d 100644 --- a/src/middleware/packages/backup/utils/fsRemove.js +++ b/src/middleware/packages/backup/utils/fsRemove.js @@ -1,7 +1,7 @@ const fs = require('fs'); const { join: pathJoin } = require('path'); -const fsRemove = async (removeFiles, remoteServer) => { +const fsRemove = async (removeFiles, subDir, remoteServer) => { await Promise.all( removeFiles .map(file => pathJoin(remoteServer.path, subDir, file)) diff --git a/src/middleware/packages/backup/utils/ftpRemove.js b/src/middleware/packages/backup/utils/ftpRemove.js index 4bb3bfedd..6176a13ba 100644 --- a/src/middleware/packages/backup/utils/ftpRemove.js +++ b/src/middleware/packages/backup/utils/ftpRemove.js @@ -1,4 +1,3 @@ -const fs = require('fs'); const Client = require('ssh2-sftp-client'); const { join: pathJoin } = require('path'); diff --git a/src/middleware/packages/backup/utils/rsyncCopy.js b/src/middleware/packages/backup/utils/rsyncCopy.js index d9ce46dc8..573075266 100644 --- a/src/middleware/packages/backup/utils/rsyncCopy.js +++ b/src/middleware/packages/backup/utils/rsyncCopy.js @@ -1,7 +1,7 @@ const Rsync = require('rsync'); const { join: pathJoin } = require('path'); -const rsyncCopy = (path, subDir, remoteServer) => { +const rsyncCopy = (path, subDir, remoteServer, syncDelete = false) => { // Setup rsync to remote server const rsync = new Rsync() .flags('arv') @@ -9,6 +9,8 @@ const rsyncCopy = (path, subDir, remoteServer) => { .source(path) .destination(`${remoteServer.user}@${remoteServer.host}:${pathJoin(remoteServer.path, subDir)}`); + if (syncDelete) rsync.set('delete'); + return new Promise((resolve, reject) => { console.log(`Rsync started with command: ${rsync.command()}`); rsync.execute(error => { diff --git a/src/middleware/packages/triplestore/subservices/dataset.js b/src/middleware/packages/triplestore/subservices/dataset.js index b9053ac77..3f6a06fb8 100644 --- a/src/middleware/packages/triplestore/subservices/dataset.js +++ b/src/middleware/packages/triplestore/subservices/dataset.js @@ -140,16 +140,18 @@ const DatasetService = { throw new Error(`Failed to delete dataset ${dataset}: ${response.statusText}`); } - // If this is a secure dataset, we might need to delete stuff manually. + // If this is a secure dataset, we need to delete stuff manually. if (isSecure) { const dbDir = path.join(this.settings.fusekiBase, 'databases', dataset); const dbAclDir = path.join(this.settings.fusekiBase, 'databases', `${dataset}Acl`); + const dbMirrorDir = path.join(this.settings.fusekiBase, 'databases', `${dataset}Mirror`); const confFile = path.join(this.settings.fusekiBase, 'configuration', `${dataset}.ttl`); // Delete all, if present. await Promise.all([ fs.promises.rm(dbDir, { recursive: true, force: true }), fs.promises.rm(dbAclDir, { recursive: true, force: true }), + fs.promises.rm(dbMirrorDir, { recursive: true, force: true }), fs.promises.rm(confFile, { force: true }) ]); } From 48b00a39301b5a0dd3342acf6082482e5c07b009 Mon Sep 17 00:00:00 2001 From: Laurin Weger <59744144+Laurin-W@users.noreply.github.com> Date: Wed, 29 May 2024 15:07:23 +0200 Subject: [PATCH 6/8] filter deleted accounts on auth.account.find --- src/middleware/packages/auth/services/account.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/middleware/packages/auth/services/account.js b/src/middleware/packages/auth/services/account.js index 6bdff69b6..76eef17e6 100644 --- a/src/middleware/packages/auth/services/account.js +++ b/src/middleware/packages/auth/services/account.js @@ -88,6 +88,12 @@ module.exports = { const accounts = await this._find(ctx, { query: { email } }); return accounts.length > 0; }, + /** Overwrite find method, to filter accounts with tombstone. */ + async find(ctx) { + /** @type {object[]} */ + const accounts = await this._find(ctx, ctx.params); + return accounts.filter(account => !account.deletedAt); + }, async findByUsername(ctx) { const { username } = ctx.params; const accounts = await this._find(ctx, { query: { username } }); From 2ecaa3b90ab44266abc3efc1bbb00bc591214cfc Mon Sep 17 00:00:00 2001 From: Laurin Weger <59744144+Laurin-W@users.noreply.github.com> Date: Thu, 30 May 2024 15:12:31 +0200 Subject: [PATCH 7/8] listBackupsFroDataset action --- src/middleware/packages/backup/index.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/middleware/packages/backup/index.js b/src/middleware/packages/backup/index.js index 2533d9d37..84ce774ff 100644 --- a/src/middleware/packages/backup/index.js +++ b/src/middleware/packages/backup/index.js @@ -122,11 +122,7 @@ const BackupService = { ); } - // File format: _ - const backupsPattern = RegExp(`^${dataset}_.{10}_.{8}\\.nq\\.gz$`); - const deleteFilenames = await fs.promises - .readdir(pathJoin(fusekiBase, 'backups')) - .then(files => files.filter(file => backupsPattern.test(file))); + const deleteFilenames = await ctx.call('backup.listBackupsForDataset', { dataset }); // Delete all backups locally. await Promise.all( @@ -152,6 +148,17 @@ const BackupService = { throw new Error(`Unknown copy method: ${copyMethod}`); } } + }, + async listBackupsForDataset(ctx) { + const { dataset } = ctx.params; + + // File format: _ + const backupsPattern = RegExp(`^${dataset}_.{10}_.{8}\\.nq\\.gz$`); + const filenames = await fs.promises + .readdir(pathJoin(this.settings.localServer.fusekiBase, 'backups')) + .then(files => files.filter(file => backupsPattern.test(file))); + + return filenames; } } }; From ba0064f947bb915ac25938c2cf0868472e4cc676 Mon Sep 17 00:00:00 2001 From: Laurin Weger <59744144+Laurin-W@users.noreply.github.com> Date: Wed, 5 Jun 2024 16:25:35 +0200 Subject: [PATCH 8/8] adjustment --- src/middleware/packages/backup/index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/middleware/packages/backup/index.js b/src/middleware/packages/backup/index.js index 84ce774ff..9560d5581 100644 --- a/src/middleware/packages/backup/index.js +++ b/src/middleware/packages/backup/index.js @@ -125,9 +125,7 @@ const BackupService = { const deleteFilenames = await ctx.call('backup.listBackupsForDataset', { dataset }); // Delete all backups locally. - await Promise.all( - deleteFilenames.map(file => pathJoin(fusekiBase, 'backups', file)).map(file => fs.promises.rm(file)) - ); + await Promise.all(deleteFilenames.map(file => fs.promises.rm(file))); // Delete backups from remote.fusekiBase switch (copyMethod) { @@ -149,6 +147,7 @@ const BackupService = { } } }, + /** Returns an array of file paths to the backups relative to `this.settings.localServer.fusekiBase`. */ async listBackupsForDataset(ctx) { const { dataset } = ctx.params; @@ -156,7 +155,8 @@ const BackupService = { const backupsPattern = RegExp(`^${dataset}_.{10}_.{8}\\.nq\\.gz$`); const filenames = await fs.promises .readdir(pathJoin(this.settings.localServer.fusekiBase, 'backups')) - .then(files => files.filter(file => backupsPattern.test(file))); + .then(files => files.filter(file => backupsPattern.test(file))) + .then(files => files.map(file => pathJoin(this.settings.localServer.fusekiBase, 'backups', file))); return filenames; }