From 1146746d9a692a506e9f5e42225e763ffeda3217 Mon Sep 17 00:00:00 2001 From: Guilherme Gabriel Date: Mon, 15 Jan 2024 15:51:33 -0300 Subject: [PATCH 1/6] add: lastLogin in model and update when user login --- src/index.ts | 10 ++++++++++ src/models/user.model.ts | 2 ++ src/server/middlewares/auth.ts | 16 +++++++++++++++- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index df915c662..ac3528763 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,6 +8,7 @@ import config from 'config'; import { logger } from './services/logger.service'; import { checkConfig } from '@utils/server/checkConfig.util'; import buildSchema from '@utils/schema/buildSchema'; +import { CronJob } from 'cron'; // Needed for survey.model, as xmlhttprequest is not defined in servers global.XMLHttpRequest = require('xhr2'); @@ -53,4 +54,13 @@ const launchServer = async () => { }); }; +// cron job to run every week +new CronJob( + '0 * * * * *', // Runs every minute + () => { + // logger.info('Running cron job', new Date()); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any +).start(); + launchServer(); diff --git a/src/models/user.model.ts b/src/models/user.model.ts index fa384e4d7..79538430e 100644 --- a/src/models/user.model.ts +++ b/src/models/user.model.ts @@ -34,6 +34,7 @@ const userSchema = new Schema( type: mongoose.Schema.Types.Mixed, }, deleteAt: { type: Date, expires: 0 }, // Date of when we must remove the user + lastLogin: Date, }, { timestamps: { createdAt: 'createdAt', updatedAt: 'modifiedAt' }, @@ -56,6 +57,7 @@ export interface User extends Document { attributes?: any; modifiedAt?: Date; deleteAt?: Date; + lastLogin?: Date; } userSchema.index( diff --git a/src/server/middlewares/auth.ts b/src/server/middlewares/auth.ts index 9a818626e..8ab263e8d 100644 --- a/src/server/middlewares/auth.ts +++ b/src/server/middlewares/auth.ts @@ -59,6 +59,7 @@ if (config.get('auth.provider') === AuthenticationType.keycloak) { user.firstName = token.given_name; user.lastName = token.family_name; user.name = token.name; + user.lastLogin = new Date(); user.oid = token.sub; user.deleteAt = undefined; // deactivate the planned deletion user @@ -77,6 +78,7 @@ if (config.get('auth.provider') === AuthenticationType.keycloak) { if (!user.lastName) { user.lastName = token.family_name; } + user.lastLogin = new Date(); user .save() .then(() => { @@ -86,7 +88,15 @@ if (config.get('auth.provider') === AuthenticationType.keycloak) { userAuthCallback(err2, done, token, user); }); } else { - userAuthCallback(null, done, token, user); + user.lastLogin = new Date(); + user + .save() + .then(() => { + userAuthCallback(null, done, token, user); + }) + .catch((err2) => { + userAuthCallback(err2, done, token, user); + }); } } } else { @@ -97,6 +107,7 @@ if (config.get('auth.provider') === AuthenticationType.keycloak) { username: token.email, name: token.name, oid: token.sub, + lastLogin: new Date(), roles: [], positionAttributes: [], }); @@ -171,6 +182,7 @@ if (config.get('auth.provider') === AuthenticationType.keycloak) { user.firstName = token.given_name; user.lastName = token.family_name; user.name = token.name; + user.lastLogin = new Date(); user.oid = token.oid; updateUser(user, req).then(() => { user @@ -191,6 +203,7 @@ if (config.get('auth.provider') === AuthenticationType.keycloak) { if (!user.lastName) { user.lastName = token.family_name; } + user.lastLogin = new Date(); user .save() .then(() => { @@ -211,6 +224,7 @@ if (config.get('auth.provider') === AuthenticationType.keycloak) { lastName: token.family_name, username: token.preferred_username, name: token.name, + lastLogin: new Date(), oid: token.oid, roles: [], positionAttributes: [], From a7c0ae4f50531e11579df03baa37ee45e5068a55 Mon Sep 17 00:00:00 2001 From: Guilherme Gabriel Date: Tue, 16 Jan 2024 10:48:40 -0300 Subject: [PATCH 2/6] feat: add cron job that delete unwanted data --- migrations/1705411074596-add-last-login.ts | 29 +++++ src/index.ts | 143 ++++++++++++++++++++- 2 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 migrations/1705411074596-add-last-login.ts diff --git a/migrations/1705411074596-add-last-login.ts b/migrations/1705411074596-add-last-login.ts new file mode 100644 index 000000000..7b4c7e620 --- /dev/null +++ b/migrations/1705411074596-add-last-login.ts @@ -0,0 +1,29 @@ +import { User } from '@models'; +import { startDatabaseForMigration } from '../src/utils/migrations/database.helper'; + +/** + * Update lastLogin field. + */ +export const up = async () => { + await startDatabaseForMigration(); + + const users = await User.find({}); + for (const user of users) { + // Create field lastLogin with value of modifiedAt + user.lastLogin = user.modifiedAt; + + // Save user + await user.save(); + } +}; + +/** + * Sample function of down migration + * + * @returns just migrate data. + */ +export const down = async () => { + /* + Code you downgrade script here! + */ +}; diff --git a/src/index.ts b/src/index.ts index ac3528763..69126531b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,7 @@ import { logger } from './services/logger.service'; import { checkConfig } from '@utils/server/checkConfig.util'; import buildSchema from '@utils/schema/buildSchema'; import { CronJob } from 'cron'; +import { Record, Resource, User } from '@models'; // Needed for survey.model, as xmlhttprequest is not defined in servers global.XMLHttpRequest = require('xhr2'); @@ -54,11 +55,147 @@ const launchServer = async () => { }); }; -// cron job to run every week +// Cron job to run every week new CronJob( - '0 * * * * *', // Runs every minute + // '0 * * * * *', // Runs every minute + '0 0 * * 0', // Runs every week () => { - // logger.info('Running cron job', new Date()); + (async function clearDataGDPR() { + const IS_ALIMENTAIDE = + config.get('server.url') === + 'https://alimentaide-973-guyane.oortcloud.tech/api'; + + // Check if the server is GUYANE + if (!IS_ALIMENTAIDE) { + return; + } + + // Get all users with lastLogin 6 months ago + const usersToDelete = await User.find({ + lastLogin: { $lt: new Date(Date.now() - 6 * 30 * 24 * 60 * 60 * 1000) }, // 6 months ago + }); + + // Find the resource with name staff + const staff = await Resource.findOne({ name: 'Staff' }); + + // Find all the records of staff with the users found above + const usersStaffRecords = await Record.find({ + resource: staff._id, + user: { $in: usersToDelete.map((user) => user._id) }, + }); + + // Hiding the info both on the user and their staff record + // username/email: [RANDOM STRING]@anonymus-oort.com + // name: [ANONYMOUS] + + // Hide the info on the user + for (let i = 0; i < usersToDelete.length; i++) { + const user = usersToDelete[i]; + + // Update the user + user.username = `${Math.random() + .toString(36) + .substring(2, 15)}@anonymus-oort.com`; + user.firstName = 'ANONYMOUS'; + user.lastName = 'ANONYMOUS'; + user.name = 'ANONYMOUS'; + user.roles = []; + + await user.save(); + } + + // Hide the info on the staff record + for (let i = 0; i < usersStaffRecords.length; i++) { + const staffRecord = usersStaffRecords[i]; + + // Update the staff record + staffRecord._createdBy = new User({ + name: 'ANONYMOUS', + username: `${Math.random() + .toString(36) + .substring(2, 15)}@anonymus-oort.com`, + }); + staffRecord.data.nom_employes = 'ANONYMOUS'; + staffRecord.data.prenom_employes = 'ANONYMOUS'; + staffRecord.data.nom_prenom_employes = 'ANONYMOUS'; + staffRecord.data.tel_staff = 'ANONYMOUS'; + staffRecord.data.email_staff = 'ANONYMOUS'; + staffRecord.data.birthdate_employes = 'ANONYMOUS'; + staffRecord._lastUpdatedBy = new User({ + name: 'ANONYMOUS', + username: `${Math.random() + .toString(36) + .substring(2, 15)}@anonymus-oort.com`, + }); + staffRecord.data.file_gdpr_staff = []; + + await staffRecord.save(); + } + + // For the beneficiaries + // Trigger when: The last aid to the family was given more than 18 months ago + // Than: Anonymize all members of that family + + // Find the resource with name Aid + const Aid = await Resource.findOne({ name: 'Aid' }); + + // Find all the records of Ais was given more than 18 months ago + const AidRecords = await Record.find({ + resource: Aid._id, + createdAt: { + $lt: new Date(Date.now() - 18 * 30 * 24 * 60 * 60 * 1000), + }, // 18 months ago + }); + + // Anonymize all members of that family + for (let i = 0; i < AidRecords.length; i++) { + const aidRecord = AidRecords[i]; + + // Get Family record + const familyRecord = await Record.findOne({ + _id: aidRecord?.data?.owner_resource, + }); + + // Find all members of the family + const members = await Record.find({ + _id: { $in: familyRecord?.data?.members }, + }); + + // Anonymize all members of that family + for (let j = 0; j < members.length; j++) { + const member = members[j]; + + // Anonymize the member + member._createdBy = new User({ + name: 'ANONYMOUS', + username: `${Math.random() + .toString(36) + .substring(2, 15)}@anonymus-oort.com`, + }); + member.data.location = 'ANONYMOUS'; + member.data.surname = 'ANONYMOUS'; + member.data.firstname = 'ANONYMOUS'; + member.data.phone = 'ANONYMOUS'; + member.data.nom_employes = 'ANONYMOUS'; + member.data.gender = 'ANONYMOUS'; + member.data.birthdate = 'ANONYMOUS'; + member.data.prenom_employes = 'ANONYMOUS'; + member.data.nom_prenom_employes = 'ANONYMOUS'; + member.data.tel_staff = 'ANONYMOUS'; + member.data.email_staff = 'ANONYMOUS'; + member.data.birthdate_employes = 'ANONYMOUS'; + member._lastUpdatedBy = new User({ + name: 'ANONYMOUS', + username: `${Math.random() + .toString(36) + .substring(2, 15)}@anonymus-oort.com`, + }); + member.data.file_gdpr_staff = []; + + await member.save(); + } + } + })(); } // eslint-disable-next-line @typescript-eslint/no-explicit-any ).start(); From 8e7a8b2d9190972eee783a215009338087cf339d Mon Sep 17 00:00:00 2001 From: matheus-relief Date: Sun, 21 Jan 2024 01:58:09 -0300 Subject: [PATCH 3/6] refactor + few improvements --- migrations/1705411074596-add-last-login.ts | 29 ---- src/index.ts | 151 +-------------------- src/jobs/anonymizeBeneficiaries.ts | 65 +++++++++ src/jobs/anonymizeStaff.ts | 78 +++++++++++ src/jobs/index.ts | 54 ++++++++ src/jobs/utils/posterizeAge.ts | 63 +++++++++ src/utils/files/deleteFile.ts | 35 +++++ src/utils/files/index.ts | 1 + 8 files changed, 300 insertions(+), 176 deletions(-) delete mode 100644 migrations/1705411074596-add-last-login.ts create mode 100644 src/jobs/anonymizeBeneficiaries.ts create mode 100644 src/jobs/anonymizeStaff.ts create mode 100644 src/jobs/index.ts create mode 100644 src/jobs/utils/posterizeAge.ts create mode 100644 src/utils/files/deleteFile.ts diff --git a/migrations/1705411074596-add-last-login.ts b/migrations/1705411074596-add-last-login.ts deleted file mode 100644 index 7b4c7e620..000000000 --- a/migrations/1705411074596-add-last-login.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { User } from '@models'; -import { startDatabaseForMigration } from '../src/utils/migrations/database.helper'; - -/** - * Update lastLogin field. - */ -export const up = async () => { - await startDatabaseForMigration(); - - const users = await User.find({}); - for (const user of users) { - // Create field lastLogin with value of modifiedAt - user.lastLogin = user.modifiedAt; - - // Save user - await user.save(); - } -}; - -/** - * Sample function of down migration - * - * @returns just migrate data. - */ -export const down = async () => { - /* - Code you downgrade script here! - */ -}; diff --git a/src/index.ts b/src/index.ts index 69126531b..b25a79720 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,8 +8,7 @@ import config from 'config'; import { logger } from './services/logger.service'; import { checkConfig } from '@utils/server/checkConfig.util'; import buildSchema from '@utils/schema/buildSchema'; -import { CronJob } from 'cron'; -import { Record, Resource, User } from '@models'; +import { startJobs } from 'jobs'; // Needed for survey.model, as xmlhttprequest is not defined in servers global.XMLHttpRequest = require('xhr2'); @@ -53,151 +52,9 @@ const launchServer = async () => { logger.info(`🚀 Server ready at ws://localhost:${PORT}/graphql`); }); }); -}; - -// Cron job to run every week -new CronJob( - // '0 * * * * *', // Runs every minute - '0 0 * * 0', // Runs every week - () => { - (async function clearDataGDPR() { - const IS_ALIMENTAIDE = - config.get('server.url') === - 'https://alimentaide-973-guyane.oortcloud.tech/api'; - - // Check if the server is GUYANE - if (!IS_ALIMENTAIDE) { - return; - } - - // Get all users with lastLogin 6 months ago - const usersToDelete = await User.find({ - lastLogin: { $lt: new Date(Date.now() - 6 * 30 * 24 * 60 * 60 * 1000) }, // 6 months ago - }); - - // Find the resource with name staff - const staff = await Resource.findOne({ name: 'Staff' }); - - // Find all the records of staff with the users found above - const usersStaffRecords = await Record.find({ - resource: staff._id, - user: { $in: usersToDelete.map((user) => user._id) }, - }); - - // Hiding the info both on the user and their staff record - // username/email: [RANDOM STRING]@anonymus-oort.com - // name: [ANONYMOUS] - - // Hide the info on the user - for (let i = 0; i < usersToDelete.length; i++) { - const user = usersToDelete[i]; - - // Update the user - user.username = `${Math.random() - .toString(36) - .substring(2, 15)}@anonymus-oort.com`; - user.firstName = 'ANONYMOUS'; - user.lastName = 'ANONYMOUS'; - user.name = 'ANONYMOUS'; - user.roles = []; - - await user.save(); - } - - // Hide the info on the staff record - for (let i = 0; i < usersStaffRecords.length; i++) { - const staffRecord = usersStaffRecords[i]; - - // Update the staff record - staffRecord._createdBy = new User({ - name: 'ANONYMOUS', - username: `${Math.random() - .toString(36) - .substring(2, 15)}@anonymus-oort.com`, - }); - staffRecord.data.nom_employes = 'ANONYMOUS'; - staffRecord.data.prenom_employes = 'ANONYMOUS'; - staffRecord.data.nom_prenom_employes = 'ANONYMOUS'; - staffRecord.data.tel_staff = 'ANONYMOUS'; - staffRecord.data.email_staff = 'ANONYMOUS'; - staffRecord.data.birthdate_employes = 'ANONYMOUS'; - staffRecord._lastUpdatedBy = new User({ - name: 'ANONYMOUS', - username: `${Math.random() - .toString(36) - .substring(2, 15)}@anonymus-oort.com`, - }); - staffRecord.data.file_gdpr_staff = []; - await staffRecord.save(); - } - - // For the beneficiaries - // Trigger when: The last aid to the family was given more than 18 months ago - // Than: Anonymize all members of that family - - // Find the resource with name Aid - const Aid = await Resource.findOne({ name: 'Aid' }); - - // Find all the records of Ais was given more than 18 months ago - const AidRecords = await Record.find({ - resource: Aid._id, - createdAt: { - $lt: new Date(Date.now() - 18 * 30 * 24 * 60 * 60 * 1000), - }, // 18 months ago - }); - - // Anonymize all members of that family - for (let i = 0; i < AidRecords.length; i++) { - const aidRecord = AidRecords[i]; - - // Get Family record - const familyRecord = await Record.findOne({ - _id: aidRecord?.data?.owner_resource, - }); - - // Find all members of the family - const members = await Record.find({ - _id: { $in: familyRecord?.data?.members }, - }); - - // Anonymize all members of that family - for (let j = 0; j < members.length; j++) { - const member = members[j]; - - // Anonymize the member - member._createdBy = new User({ - name: 'ANONYMOUS', - username: `${Math.random() - .toString(36) - .substring(2, 15)}@anonymus-oort.com`, - }); - member.data.location = 'ANONYMOUS'; - member.data.surname = 'ANONYMOUS'; - member.data.firstname = 'ANONYMOUS'; - member.data.phone = 'ANONYMOUS'; - member.data.nom_employes = 'ANONYMOUS'; - member.data.gender = 'ANONYMOUS'; - member.data.birthdate = 'ANONYMOUS'; - member.data.prenom_employes = 'ANONYMOUS'; - member.data.nom_prenom_employes = 'ANONYMOUS'; - member.data.tel_staff = 'ANONYMOUS'; - member.data.email_staff = 'ANONYMOUS'; - member.data.birthdate_employes = 'ANONYMOUS'; - member._lastUpdatedBy = new User({ - name: 'ANONYMOUS', - username: `${Math.random() - .toString(36) - .substring(2, 15)}@anonymus-oort.com`, - }); - member.data.file_gdpr_staff = []; - - await member.save(); - } - } - })(); - } - // eslint-disable-next-line @typescript-eslint/no-explicit-any -).start(); + // Start all the custom jobs + startJobs(); +}; launchServer(); diff --git a/src/jobs/anonymizeBeneficiaries.ts b/src/jobs/anonymizeBeneficiaries.ts new file mode 100644 index 000000000..b11d379d5 --- /dev/null +++ b/src/jobs/anonymizeBeneficiaries.ts @@ -0,0 +1,65 @@ +import { Record, User } from '@models'; +import { Types } from 'mongoose'; + +/** Staff resource ID */ +const AID_RESOURCE_ID = new Types.ObjectId('64e6e0933c7bf3962bf4f04c'); + +/** Anonymizes the beneficiary data, if didn't log in for more than 18 months */ +export const anonymizeStaff = async () => { + // Find all the records of Ais was given more than 18 months ago + const allAids = await Record.find({ + resource: AID_RESOURCE_ID, + createdAt: { + $lt: new Date(Date.now() - 18 * 30 * 24 * 60 * 60 * 1000), + }, // 18 months ago + }); + + // Anonymize all members of that family + for (let i = 0; i < allAids.length; i++) { + const aidRecord = allAids[i]; + + // Get Family record + const familyRecord = await Record.findOne({ + _id: aidRecord?.data?.owner_resource, + }); + + // Find all members of the family + const members = await Record.find({ + _id: { $in: familyRecord?.data?.members }, + }); + + // Anonymize all members of that family + for (let j = 0; j < members.length; j++) { + const member = members[j]; + + // Anonymize the member + member._createdBy = new User({ + name: 'ANONYMOUS', + username: `${Math.random() + .toString(36) + .substring(2, 15)}@anonymus-oort.com`, + }); + member.data.location = 'ANONYMOUS'; + member.data.surname = 'ANONYMOUS'; + member.data.firstname = 'ANONYMOUS'; + member.data.phone = 'ANONYMOUS'; + member.data.nom_employes = 'ANONYMOUS'; + member.data.gender = 'ANONYMOUS'; + member.data.birthdate = 'ANONYMOUS'; + member.data.prenom_employes = 'ANONYMOUS'; + member.data.nom_prenom_employes = 'ANONYMOUS'; + member.data.tel_staff = 'ANONYMOUS'; + member.data.email_staff = 'ANONYMOUS'; + member.data.birthdate_employes = 'ANONYMOUS'; + member._lastUpdatedBy = new User({ + name: 'ANONYMOUS', + username: `${Math.random() + .toString(36) + .substring(2, 15)}@anonymus-oort.com`, + }); + member.data.file_gdpr_staff = []; + + await member.save(); + } + } +}; diff --git a/src/jobs/anonymizeStaff.ts b/src/jobs/anonymizeStaff.ts new file mode 100644 index 000000000..4eda17a40 --- /dev/null +++ b/src/jobs/anonymizeStaff.ts @@ -0,0 +1,78 @@ +import { Record, User } from '@models'; +import { deleteFile } from '@utils/files'; +import { Types } from 'mongoose'; +import { posterizeAge } from './utils/posterizeAge'; +import { logger } from '@services/logger.service'; + +/** Staff resource ID */ +const STAFF_RESOURCE_ID = new Types.ObjectId('649e9ec5eae9f89219921eff'); + +/** Anonymizes the staff data, if didn't log in for more than 6 months */ +export const anonymizeStaff = async () => { + // Get all users with lastLogin 6 months ago + const usersToDelete = await User.find({ + $expr: { + $lt: [ + { + $ifNull: ['$lastLogin', '$modifiedAt'], + }, + new Date(Date.now() - 6 * 30 * 24 * 60 * 60 * 1000), // 6 months ago + ], + }, + }); + + // Find all the records of staff with the users found above + const usersStaffRecords = await Record.find({ + resource: STAFF_RESOURCE_ID, + user: { $in: usersToDelete.map((user) => user._id) }, + }); + + // Hide the info on the user + usersToDelete.forEach((user) => { + user.username = `${user._id.toString()}@oort-anonymous.com`; + user.firstName = 'ANONYMOUS'; + user.lastName = 'ANONYMOUS'; + user.name = 'ANONYMOUS'; + user.roles = []; + user.oid = null; + }); + + await User.bulkSave(usersToDelete); + + // Hold all files that should be deleted in blob storage + const filesToDelete = []; + + // Hide info on the staff record + usersStaffRecords.forEach((staffRecord) => { + if (!staffRecord.data) { + return; + } + + // Add all files to delete + (staffRecord.data.file_gdpr_staff || []).forEach((file) => { + filesToDelete.push(file.content); + }); + + staffRecord.data = { + ...staffRecord.data, + nom_employes: 'ANONYMOUS', + prenom_employes: 'ANONYMOUS', + tel_staff: 'ANONYMOUS', + email_staff: `${staffRecord.data.linked_user[0]}@oort-anonymous.com`, + file_gdpr_staff: [], + birthdate_employes: posterizeAge({ + birthdate: staffRecord.data.birthdate_employes, + }), + }; + }); + + // Delete all files + Promise.all(filesToDelete.map((file) => deleteFile('forms', file))).catch( + (err) => { + logger.error(`Error deleting files: ${err}`); + } + ); + + // Save all the records + await Record.bulkSave(usersStaffRecords); +}; diff --git a/src/jobs/index.ts b/src/jobs/index.ts new file mode 100644 index 000000000..fe41b3d05 --- /dev/null +++ b/src/jobs/index.ts @@ -0,0 +1,54 @@ +import { CronJob } from 'cron'; +import { anonymizeStaff } from './anonymizeStaff'; +import { logger } from '@services/logger.service'; +import config from 'config'; + +/** All available jobs */ +const JOBS: { + name: string; + description: string; + fn: () => Promise; + // Schedule in cron format + schedule: string; + // Environments where the job should be started + envs: string[]; +}[] = [ + { + name: 'Anonymize staff', + description: "Anonymizes staff, if didn't log in for more than 6 months", + // Every week + schedule: '0 0 * * 0', + fn: anonymizeStaff, + envs: ['alimentaide'], + }, +]; + +/** Starts all the jobs */ +export const startJobs = () => { + const isDev = config.util.getEnv('NODE_ENV') === 'development'; + const env = config.util.getEnv('NODE_CONFIG_ENV'); + + // Start all the jobs + JOBS.forEach((job) => { + // Check if the job should be started + if (!isDev && !job.envs.includes(env)) { + return; + } + + // Start the job + new CronJob( + job.schedule, + async () => { + try { + await job.fn(); + } catch (error) { + console.error(error); + } + }, + null, + true + ).start(); + + logger.info(`🤖 Job "${job.name}" started`); + }); +}; diff --git a/src/jobs/utils/posterizeAge.ts b/src/jobs/utils/posterizeAge.ts new file mode 100644 index 000000000..9cbcac1ca --- /dev/null +++ b/src/jobs/utils/posterizeAge.ts @@ -0,0 +1,63 @@ +/** Defined age groups */ +const AGE_GROUPS = [ + [0, 3], + [4, 14], + [15, 17], + [18, 25], + [26, 64], + [65, 79], + [80, null], +]; + +/** + * Get the age of a person + * + * @param birthdate Date of birth of the person + * @returns The age of the person + */ +const getAge = (birthdate: Date) => { + const today = new Date(); + const birthDate = new Date(birthdate); + let age = today.getFullYear() - birthDate.getFullYear(); + const m = today.getMonth() - birthDate.getMonth(); + if (m < 0 || (m === 0 && today.getDate() < birthDate.getDate())) { + age--; + } + return age; +}; + +/** + * Gets the age group of a person + * + * @param param Params object + * @param param.age Age of the person + * @param param.birthdate Birthdate of the person + * @returns The age group of the person + */ +export const posterizeAge = ({ + age, + birthdate, +}: { + age?: number; + birthdate?: string; +}): number | string | null => { + if (age && typeof age === 'number') { + // Find the age group + const ageGroup = AGE_GROUPS.find( + (group) => age >= group[0] && (!group[1] || age <= group[1]) + ); + + // Get random age in the age group + const min = ageGroup[0]; + const max = ageGroup[1] || 100; + return Math.floor(Math.random() * (max - min + 1)) + min; + } else if (birthdate && !isNaN(new Date(birthdate).getTime())) { + const newAge = posterizeAge({ age: getAge(new Date(birthdate)) }) as number; + + // Random month and day + return `${new Date().getFullYear() - newAge}-${ + Math.floor(Math.random() * 11) + 1 + }-${Math.floor(Math.random() * 27) + 1}`; + } + return null; +}; diff --git a/src/utils/files/deleteFile.ts b/src/utils/files/deleteFile.ts new file mode 100644 index 000000000..04415e509 --- /dev/null +++ b/src/utils/files/deleteFile.ts @@ -0,0 +1,35 @@ +import { BlobServiceClient } from '@azure/storage-blob'; +import { logger } from '@services/logger.service'; +import { GraphQLError } from 'graphql'; +import i18next from 'i18next'; +import config from 'config'; + +/** Azure storage connection string */ +const AZURE_STORAGE_CONNECTION_STRING: string = config.get( + 'blobStorage.connectionString' +); + +/** + * Delete a file in Azure storage. + * + * @param containerName main container name + * @param path path to the blob + */ +export const deleteFile = async ( + containerName: string, + path: string +): Promise => { + try { + const blobServiceClient = BlobServiceClient.fromConnectionString( + AZURE_STORAGE_CONNECTION_STRING + ); + const containerClient = blobServiceClient.getContainerClient(containerName); + const file = containerClient.getBlockBlobClient(path); + await file.deleteIfExists(); + } catch (err) { + logger.error(err.message, { stack: err.stack }); + throw new GraphQLError( + i18next.t('utils.files.uploadFile.errors.fileCannotBeDeleted') + ); + } +}; diff --git a/src/utils/files/index.ts b/src/utils/files/index.ts index 83ce4875c..b730181c9 100644 --- a/src/utils/files/index.ts +++ b/src/utils/files/index.ts @@ -1,5 +1,6 @@ export * from './fileBuilder'; export * from './uploadFile'; +export * from './deleteFile'; export * from './downloadFile'; export * from './templateBuilder'; export * from './getColumns'; From b915416873de9adda19798b2d4c7e5c5afb368a8 Mon Sep 17 00:00:00 2001 From: matheus-relief Date: Sun, 21 Jan 2024 02:07:09 -0300 Subject: [PATCH 4/6] minor fixes --- src/jobs/anonymizeBeneficiaries.ts | 2 +- src/jobs/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jobs/anonymizeBeneficiaries.ts b/src/jobs/anonymizeBeneficiaries.ts index b11d379d5..da1cd99ad 100644 --- a/src/jobs/anonymizeBeneficiaries.ts +++ b/src/jobs/anonymizeBeneficiaries.ts @@ -5,7 +5,7 @@ import { Types } from 'mongoose'; const AID_RESOURCE_ID = new Types.ObjectId('64e6e0933c7bf3962bf4f04c'); /** Anonymizes the beneficiary data, if didn't log in for more than 18 months */ -export const anonymizeStaff = async () => { +export const anonymizeBeneficiaries = async () => { // Find all the records of Ais was given more than 18 months ago const allAids = await Record.find({ resource: AID_RESOURCE_ID, diff --git a/src/jobs/index.ts b/src/jobs/index.ts index fe41b3d05..07994346e 100644 --- a/src/jobs/index.ts +++ b/src/jobs/index.ts @@ -42,7 +42,7 @@ export const startJobs = () => { try { await job.fn(); } catch (error) { - console.error(error); + logger.error(error); } }, null, From b82e95b007f0713f7d2d997c647802ea84c29391 Mon Sep 17 00:00:00 2001 From: Guilherme Gabriel Date: Tue, 23 Jan 2024 01:25:48 -0300 Subject: [PATCH 5/6] fix: beneficiaries logic --- src/jobs/anonymizeBeneficiaries.ts | 101 ++++++++++++++++------------- src/jobs/index.ts | 9 +++ 2 files changed, 64 insertions(+), 46 deletions(-) diff --git a/src/jobs/anonymizeBeneficiaries.ts b/src/jobs/anonymizeBeneficiaries.ts index da1cd99ad..b759a26bc 100644 --- a/src/jobs/anonymizeBeneficiaries.ts +++ b/src/jobs/anonymizeBeneficiaries.ts @@ -3,63 +3,72 @@ import { Types } from 'mongoose'; /** Staff resource ID */ const AID_RESOURCE_ID = new Types.ObjectId('64e6e0933c7bf3962bf4f04c'); +const FAMILY_RESOURCE_ID = new Types.ObjectId('64de75fd3fb2a11c988dddb2'); /** Anonymizes the beneficiary data, if didn't log in for more than 18 months */ export const anonymizeBeneficiaries = async () => { - // Find all the records of Ais was given more than 18 months ago - const allAids = await Record.find({ - resource: AID_RESOURCE_ID, - createdAt: { - $lt: new Date(Date.now() - 18 * 30 * 24 * 60 * 60 * 1000), - }, // 18 months ago - }); + // For all family records, check if + // in the last 18 months they received aid - // Anonymize all members of that family - for (let i = 0; i < allAids.length; i++) { - const aidRecord = allAids[i]; + // Get all the family records + const allFamilies = await Record.find({ + resource: FAMILY_RESOURCE_ID, + }); - // Get Family record - const familyRecord = await Record.findOne({ - _id: aidRecord?.data?.owner_resource, + // For each family record, check if exists + // an aid record in the last 18 months + for (const family of allFamilies) { + const aidGivenToFamily = await Record.exists({ + resource: AID_RESOURCE_ID, + createdAt: { + $gt: new Date(Date.now() - 18 * 30 * 24 * 60 * 60 * 1000), + }, // 18 months ago + 'data.owner_resource': family._id.toString(), }); - // Find all members of the family - const members = await Record.find({ - _id: { $in: familyRecord?.data?.members }, - }); + // If no aid was given to the family in the last 18 months + if (!aidGivenToFamily) { + // Find all members of the family + const members = await Record.find({ + _id: { $in: family?.data?.members }, + }); - // Anonymize all members of that family - for (let j = 0; j < members.length; j++) { - const member = members[j]; + // Anonymize all the members + members.forEach((member) => { + if (!member.data) { + return; + } + // Anonymize the member + member._createdBy = new User({ + name: 'ANONYMOUS', + username: `${member._id.toString()}@oort-anonymous.com`, + }); - // Anonymize the member - member._createdBy = new User({ - name: 'ANONYMOUS', - username: `${Math.random() - .toString(36) - .substring(2, 15)}@anonymus-oort.com`, - }); - member.data.location = 'ANONYMOUS'; - member.data.surname = 'ANONYMOUS'; - member.data.firstname = 'ANONYMOUS'; - member.data.phone = 'ANONYMOUS'; - member.data.nom_employes = 'ANONYMOUS'; - member.data.gender = 'ANONYMOUS'; - member.data.birthdate = 'ANONYMOUS'; - member.data.prenom_employes = 'ANONYMOUS'; - member.data.nom_prenom_employes = 'ANONYMOUS'; - member.data.tel_staff = 'ANONYMOUS'; - member.data.email_staff = 'ANONYMOUS'; - member.data.birthdate_employes = 'ANONYMOUS'; - member._lastUpdatedBy = new User({ - name: 'ANONYMOUS', - username: `${Math.random() - .toString(36) - .substring(2, 15)}@anonymus-oort.com`, + member.data = { + ...member.data, + location: 'ANONYMOUS', + surname: 'ANONYMOUS', + firstname: 'ANONYMOUS', + phone: 'ANONYMOUS', + nom_employes: 'ANONYMOUS', + gender: 'ANONYMOUS', + birthdate: 'ANONYMOUS', + prenom_employes: 'ANONYMOUS', + nom_prenom_employes: 'ANONYMOUS', + tel_staff: 'ANONYMOUS', + email_staff: 'ANONYMOUS', + birthdate_employes: 'ANONYMOUS', + file_gdpr_staff: [], + }; + + member._lastUpdatedBy = new User({ + name: 'ANONYMOUS', + username: `${member._id.toString()}@oort-anonymous.com`, + }); }); - member.data.file_gdpr_staff = []; - await member.save(); + // Save all the records + await Record.bulkSave(members); } } }; diff --git a/src/jobs/index.ts b/src/jobs/index.ts index 07994346e..c0d8c9648 100644 --- a/src/jobs/index.ts +++ b/src/jobs/index.ts @@ -1,5 +1,6 @@ import { CronJob } from 'cron'; import { anonymizeStaff } from './anonymizeStaff'; +import { anonymizeBeneficiaries } from './anonymizeBeneficiaries'; import { logger } from '@services/logger.service'; import config from 'config'; @@ -21,6 +22,14 @@ const JOBS: { fn: anonymizeStaff, envs: ['alimentaide'], }, + { + name: 'Anonymize beneficiaries', + description: '', + // Every week + schedule: '0 0 * * 0', + fn: anonymizeBeneficiaries, + envs: ['alimentaide'], + }, ]; /** Starts all the jobs */ From d93ec243afc5923df55ee17a7c12685ceae3cdbc Mon Sep 17 00:00:00 2001 From: Guilherme Gabriel Date: Tue, 23 Jan 2024 01:30:32 -0300 Subject: [PATCH 6/6] fix: description Anonymize beneficiaries cronjob --- src/jobs/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/jobs/index.ts b/src/jobs/index.ts index c0d8c9648..3e993e1a7 100644 --- a/src/jobs/index.ts +++ b/src/jobs/index.ts @@ -24,7 +24,8 @@ const JOBS: { }, { name: 'Anonymize beneficiaries', - description: '', + description: + 'Anonymizes all members of a family, if the family did not receive aid for the last 18 months', // Every week schedule: '0 0 * * 0', fn: anonymizeBeneficiaries,