Merge pull request #1249 from assemblee-virtuelle/feat/gdpr-features

[Breaking] GDPR compliance features

src/middleware/packages/auth/services/account.js

@@ -88,6 +88,12 @@ module.exports = {
       const accounts = await this._find(ctx, { query: { email } });
       return accounts.length > 0;
     },
+    /** Overwrite find method, to filter accounts with tombstone. */
+    async find(ctx) {
+      /** @type {object[]} */
+      const accounts = await this._find(ctx, ctx.params);
+      return accounts.filter(account => !account.deletedAt);
+    },
     async findByUsername(ctx) {
       const { username } = ctx.params;
       const accounts = await this._find(ctx, { query: { username } });
@@ -184,6 +190,34 @@ module.exports = {
         '@id': account['@id'],
         ...params
       });
+    },
+    async deleteByWebId(ctx) {
+      const { webId } = ctx.params;
+      const account = await ctx.call('auth.account.findByWebId', { webId });
+
+      if (account) {
+        await this._remove(ctx, { id: account['@id'] });
+        return true;
+      }
+
+      return false;
+    },
+    // Remove email and password from an account, set deletedAt timestamp.
+    async setTombstone(ctx) {
+      const { webId } = ctx.params;
+      const account = await ctx.call('auth.account.findByWebId', { webId });
+
+      return await this._update(ctx, {
+        // Set all values to undefined...
+        ...Object.fromEntries(Object.keys(account).map(key => [key, null])),
+        '@id': account['@id'],
+        // ...except for
+        webId: account.webId,
+        username: account.username,
+        podUri: account.podUri,
+        // And add a deletedAt date.
+        deletedAt: new Date().toISOString()
+      });
     }
   },
   methods: {

src/middleware/packages/backup/index.js

@@ -1,7 +1,11 @@
 const { CronJob } = require('cron');
+const fs = require('fs');
+const pathJoin = require('path').join;
 const fsCopy = require('./utils/fsCopy');
 const ftpCopy = require('./utils/ftpCopy');
 const rsyncCopy = require('./utils/rsyncCopy');
+const ftpRemove = require('./utils/ftpRemove');
+const fsRemove = require('./utils/fsRemove');
 /**
  * @typedef {import('moleculer').Context} Context
  */
@@ -10,7 +14,7 @@ const BackupService = {
   name: 'backup',
   settings: {
     localServer: {
-      fusekiBackupsPath: null,
+      fusekiBase: null,
       otherDirsPaths: {}
     },
     copyMethod: 'rsync', // rsync, ftp, or fs
@@ -29,33 +33,36 @@ const BackupService = {
   },
   dependencies: ['triplestore'],
   started() {
-    const { cronJob } = this.settings;
+    const {
+      cronJob,
+      localServer: { fusekiBase }
+    } = this.settings;
 
     if (cronJob.time) {
       this.cronJob = new CronJob(cronJob.time, this.actions.backupAll, null, true, cronJob.timeZone);
     }
+
+    if (!fusekiBase) {
+      throw new Error('Backup service requires `localServer.fusekiBase` setting to be set to the FUSEKI_BASE path.');
+    }
   },
   actions: {
     async backupAll(ctx) {
       await this.actions.backupDatasets({}, { parentCtx: ctx });
       await this.actions.backupOtherDirs({}, { parentCtx: ctx });
     },
     async backupDatasets(ctx) {
-      const { fusekiBackupsPath } = this.settings.localServer;
-
-      if (!fusekiBackupsPath) {
-        this.logger.info('No fusekiBackupsPath defined, skipping backup...');
-        return;
-      }
-
       // Generate a new backup of all datasets
       const datasets = await ctx.call('triplestore.dataset.list');
       for (const dataset of datasets) {
         this.logger.info(`Backing up dataset: ${dataset}`);
         await ctx.call('triplestore.dataset.backup', { dataset });
       }
 
-      await this.actions.copyToRemoteServer({ path: fusekiBackupsPath, subDir: 'datasets' }, { parentCtx: ctx });
+      await this.actions.copyToRemoteServer(
+        { path: pathJoin(this.settings.localServer.fusekiBase, 'backups'), subDir: 'datasets' },
+        { parentCtx: ctx }
+      );
     },
     async backupOtherDirs(ctx) {
       const { otherDirsPaths } = this.settings.localServer;
@@ -96,6 +103,62 @@ const BackupService = {
         default:
           throw new Error(`Unknown copy method: ${copyMethod}`);
       }
+    },
+    deleteDataset: {
+      params: {
+        dataset: { type: 'string' },
+        iKnowWhatImDoing: { type: 'boolean' }
+      },
+      async handler(ctx) {
+        const { dataset, iKnowWhatImDoing } = ctx.params;
+        const {
+          copyMethod,
+          remoteServer,
+          localServer: { fusekiBase }
+        } = this.settings;
+        if (!iKnowWhatImDoing) {
+          throw new Error(
+            'Please confirm that you know what you are doing and set the `iKnowWhatImDoing` parameter to `true`.'
+          );
+        }
+
+        const deleteFilenames = await ctx.call('backup.listBackupsForDataset', { dataset });
+
+        // Delete all backups locally.
+        await Promise.all(deleteFilenames.map(file => fs.promises.rm(file)));
+
+        // Delete backups from remote.fusekiBase
+        switch (copyMethod) {
+          case 'rsync':
+            // The last param sets the --deletion argument, to sync deletions too.
+            await rsyncCopy(pathJoin(fusekiBase, 'backups'), 'datasets', remoteServer, true);
+            break;
+
+          case 'ftp':
+            await ftpRemove(deleteFilenames, remoteServer);
+            break;
+
+          case 'fs':
+            await fsRemove(deleteFilenames, 'datasets', remoteServer);
+            break;
+
+          default:
+            throw new Error(`Unknown copy method: ${copyMethod}`);
+        }
+      }
+    },
+    /** Returns an array of file paths to the backups relative to `this.settings.localServer.fusekiBase`. */
+    async listBackupsForDataset(ctx) {
+      const { dataset } = ctx.params;
+
+      // File format: <dataset-name>_<iso timestamp, but with _ instead of T and : replaced by `-`>
+      const backupsPattern = RegExp(`^${dataset}_.{10}_.{8}\\.nq\\.gz$`);
+      const filenames = await fs.promises
+        .readdir(pathJoin(this.settings.localServer.fusekiBase, 'backups'))
+        .then(files => files.filter(file => backupsPattern.test(file)))
+        .then(files => files.map(file => pathJoin(this.settings.localServer.fusekiBase, 'backups', file)));
+
+      return filenames;
     }
   }
 };

src/middleware/packages/backup/indexTypes.d.ts

@@ -2,7 +2,7 @@ import { Context, ServiceSchema, CallingOptions } from 'moleculer';
 import { CronJob } from 'cron';
 
 interface LocalServerSettings {
-  fusekiBackupsPath: string | null;
+  fusekiBase: string | null;
   otherDirsPaths: Record<string, string>;
 }
 

src/middleware/packages/backup/utils/fsRemove.js

@@ -0,0 +1,12 @@
+const fs = require('fs');
+const { join: pathJoin } = require('path');
+
+const fsRemove = async (removeFiles, subDir, remoteServer) => {
+  await Promise.all(
+    removeFiles
+      .map(file => pathJoin(remoteServer.path, subDir, file))
+      .map(file => fs.promises.rm(file, { force: true }))
+  );
+};
+
+module.exports = fsRemove;

src/middleware/packages/backup/utils/ftpRemove.js

@@ -0,0 +1,24 @@
+const Client = require('ssh2-sftp-client');
+const { join: pathJoin } = require('path');
+
+const ftpRemove = (removeFiles, remoteServer) => {
+  return new Promise((resolve, reject) => {
+    const sftp = new Client();
+    sftp
+      .connect({
+        host: remoteServer.host,
+        port: remoteServer.port,
+        username: remoteServer.user,
+        password: remoteServer.password
+      })
+      .then(async () => {
+        for (const filename of removeFiles) {
+          await sftp.delete(pathJoin(remoteServer.path, filename), true);
+        }
+        resolve();
+      })
+      .catch(e => reject(e));
+  });
+};
+
+module.exports = ftpRemove;

src/middleware/packages/backup/utils/rsyncCopy.js

@@ -1,14 +1,16 @@
 const Rsync = require('rsync');
 const { join: pathJoin } = require('path');
 
-const rsyncCopy = (path, subDir, remoteServer) => {
+const rsyncCopy = (path, subDir, remoteServer, syncDelete = false) => {
   // Setup rsync to remote server
   const rsync = new Rsync()
     .flags('arv')
     .set('e', `sshpass -p "${remoteServer.password}" ssh -o StrictHostKeyChecking=no`)
     .source(path)
     .destination(`${remoteServer.user}@${remoteServer.host}:${pathJoin(remoteServer.path, subDir)}`);
 
+  if (syncDelete) rsync.set('delete');
+
   return new Promise((resolve, reject) => {
     console.log(`Rsync started with command: ${rsync.command()}`);
     rsync.execute(error => {

src/middleware/packages/core/service.js

@@ -35,7 +35,8 @@ const CoreService = {
       url: undefined,
       user: undefined,
       password: undefined,
-      mainDataset: undefined
+      mainDataset: undefined,
+      fusekiBase: undefined
     },
     // Optional
     containers: undefined,
@@ -173,11 +174,7 @@ const CoreService = {
 
       this.broker.createService(TripleStoreService, {
         settings: {
-          url: triplestore.url,
-          user: triplestore.user,
-          password: triplestore.password,
-          mainDataset: triplestore.mainDataset,
-          ...this.settings.triplestore
+          ...triplestore
         },
         async started() {
           if (triplestore.mainDataset) {

src/middleware/packages/middlewares/index.js

@@ -47,6 +47,11 @@ const throw403 = msg => {
   throw new MoleculerError('Forbidden', 403, 'ACCESS_DENIED', { status: 'Forbidden', text: msg });
 };
 
+/** @type {(msg: string) => never} */
+const throw404 = msg => {
+  throw new MoleculerError('Forbidden', 404, 'NOT_FOUND', { status: 'Not found', text: msg });
+};
+
 const throw500 = msg => {
   throw new MoleculerError(msg, 500, 'INTERNAL_SERVER_ERROR', { status: 'Server Error', text: msg });
 };
@@ -202,5 +207,6 @@ module.exports = {
   saveDatasetMeta,
   throw400,
   throw403,
+  throw404,
   throw500
 };

src/middleware/packages/triplestore/service.js

@@ -19,12 +19,13 @@ const TripleStoreService = {
     user: null,
     password: null,
     mainDataset: null,
+    fusekiBase: null,
     // Sub-services customization
     dataset: {}
   },
   dependencies: ['jsonld'],
   async created() {
-    const { url, user, password, dataset } = this.settings;
+    const { url, user, password, dataset, fusekiBase } = this.settings;
     this.subservices = {};
 
     if (dataset !== false) {
@@ -33,6 +34,7 @@ const TripleStoreService = {
           url,
           user,
           password,
+          fusekiBase,
           ...dataset
         }
       });