From f5c6943843e66b74a2d1257388c2c7a02158e90b Mon Sep 17 00:00:00 2001 From: shreddedbacon Date: Tue, 22 Oct 2024 11:16:20 +1100 Subject: [PATCH] chore: update delete environment and project helpers to clear more stale data when removed --- services/api/src/resources/backup/sql.ts | 20 +++++- .../api/src/resources/environment/helpers.ts | 64 +++++++++++++------ services/api/src/resources/environment/sql.ts | 10 +++ services/api/src/resources/project/helpers.ts | 18 ++++++ 4 files changed, 91 insertions(+), 21 deletions(-) diff --git a/services/api/src/resources/backup/sql.ts b/services/api/src/resources/backup/sql.ts index 223c254a24..d28af324c2 100644 --- a/services/api/src/resources/backup/sql.ts +++ b/services/api/src/resources/backup/sql.ts @@ -13,6 +13,12 @@ export const Sql = { knex('env_vars') .where('project', projectId) .toString(), + selectBackupsByEnvironmentId: (environmentId: number) => + knex('environment_backup') + .where('environment', '=', environmentId) + .orderBy('created', 'desc') + .orderBy('id', 'desc') + .toString(), insertBackup: ({ id, environment, @@ -123,5 +129,17 @@ export const Sql = { 'environment.id' ) .where('environment_backup.backup_id', backupId) - .toString() + .toString(), + // delete all environments backups from backup table that match environment id + deleteBackupsByEnvironmentId: (environmentId: number) => + knex('environment_backup') + .where('environment', '=', environmentId) + .delete() + .toString(), + // delete all environments backups from backup table that match environment ids + deleteBackupsByEnvironmentIds: (environmentIds: number[]) => + knex('environment_backup') + .whereIn('environment', environmentIds) + .delete() + .toString(), }; diff --git a/services/api/src/resources/environment/helpers.ts b/services/api/src/resources/environment/helpers.ts index 874cbfb939..1645e88da5 100644 --- a/services/api/src/resources/environment/helpers.ts +++ b/services/api/src/resources/environment/helpers.ts @@ -5,6 +5,7 @@ import { query } from '../../util/db'; import { Sql } from './sql'; import { Sql as problemSql } from '../problem/sql'; import { Sql as factSql } from '../fact/sql'; +// import { Sql as backupSql } from '../backup/sql'; import { Helpers as projectHelpers } from '../project/helpers'; import { HistoryRetentionEnforcer } from '../retentionpolicy/history'; import { logger } from '../../loggers/logger'; @@ -34,6 +35,29 @@ export const Helpers = (sqlClientPool: Pool) => { deleteEnvironment: async (name: string, eid: number, pid: number) => { const environmentData = await Helpers(sqlClientPool).getEnvironmentById(eid); const projectData = await projectHelpers(sqlClientPool).getProjectById(pid); + + // attempt to run any retention policy processes before the environment is deleted + try { + // export a dump of the project, environment data, and associated task and deployment history before the environment is deleted + await HistoryRetentionEnforcer().saveEnvironmentHistoryBeforeDeletion(projectData, environmentData) + } catch (e) { + logger.error(`error running save environment history: ${e}`) + } + // purge all history for this environment, including logs and files from s3 + try { + // remove all deployments and associated files + await HistoryRetentionEnforcer().cleanupAllDeployments(projectData, environmentData) + } catch (e) { + logger.error(`error running deployment retention enforcer: ${e}`) + } + try { + // remove all tasks and associated files + await HistoryRetentionEnforcer().cleanupAllTasks(projectData, environmentData) + } catch (e) { + logger.error(`error running task retention enforcer: ${e}`) + } + + // then proceed to purge related data try { // clean up environment variables // logger.debug(`deleting environment ${name}/id:${eid}/project:${pid} environment variables`) @@ -67,30 +91,30 @@ export const Helpers = (sqlClientPool: Pool) => { problemSql.deleteProblemsForEnvironment(eid) ); - // @TODO: environment_storage, environment_backup + // delete the environment backups rows + // logger.debug(`deleting environment ${name}/id:${eid}/project:${pid} environment backups`) + // @TODO: this could be done here, but it would mean that to recover all the backup ids of a deleted environment + // in the event that an environment is "accidentally deleted" it would require accessing the bucket + // to retrieve them from the saved history export JSON dump + // this is disabled for now, but when a project is deleted, all of the backups for any environments of that project + // will have the table cleaned out to keep the database leaner + // await query( + // sqlClientPool, + // backupSql.deleteBackupsByEnvironmentId(eid) + // ); + // clean up storage data + // logger.debug(`deleting environment ${name}/id:${eid}/project:${pid} environment storage`) + // @TODO: this could be done here, but amazee.io might still use this data for environments that are deleted + // this is disabled for now, but when a project is deleted, all of the storages for any environments of that project + // will have the table cleaned out to keep the database leaner + // await query( + // sqlClientPool, + // Sql.deleteEnvironmentStorageByEnvironmentId(eid) + // ); } catch (e) { logger.error(`error cleaning up linked environment tables: ${e}`) } - try { - // export a dump of the project, environment data, and associated task and deployment history before the environment is deleted - await HistoryRetentionEnforcer().saveEnvironmentHistoryBeforeDeletion(projectData, environmentData) - } catch (e) { - logger.error(`error running save environment history: ${e}`) - } - // purge all history for this environment, including logs and files from s3 - try { - // remove all deployments and associated files - await HistoryRetentionEnforcer().cleanupAllDeployments(projectData, environmentData) - } catch (e) { - logger.error(`error running deployment retention enforcer: ${e}`) - } - try { - // remove all tasks and associated files - await HistoryRetentionEnforcer().cleanupAllTasks(projectData, environmentData) - } catch (e) { - logger.error(`error running task retention enforcer: ${e}`) - } // delete the environment // logger.debug(`deleting environment ${name}/id:${eid}/project:${pid}`) await query( diff --git a/services/api/src/resources/environment/sql.ts b/services/api/src/resources/environment/sql.ts index 639635c323..ca343d175f 100644 --- a/services/api/src/resources/environment/sql.ts +++ b/services/api/src/resources/environment/sql.ts @@ -198,4 +198,14 @@ export const Sql = { .where('project', '=', projectId) .delete() .toString(), + deleteEnvironmentStorageByEnvironmentId: (id: number) => + knex('environment_storage') + .where('environment', '=', id) + .delete() + .toString(), + deleteEnvironmentStorageByEnvironmentIds: (ids: number[]) => + knex('environment_storage') + .whereIn('environment', ids) + .delete() + .toString(), }; diff --git a/services/api/src/resources/project/helpers.ts b/services/api/src/resources/project/helpers.ts index 03c9cac153..ae7144c9ea 100644 --- a/services/api/src/resources/project/helpers.ts +++ b/services/api/src/resources/project/helpers.ts @@ -4,6 +4,7 @@ import { asyncPipe } from '@lagoon/commons/dist/util/func'; import { query } from '../../util/db'; import { Sql } from './sql'; import { Sql as environmentSql } from '../environment/sql'; +import { Sql as backupSql } from '../backup/sql'; // import { logger } from '../../loggers/logger'; export const Helpers = (sqlClientPool: Pool) => { @@ -193,6 +194,23 @@ export const Helpers = (sqlClientPool: Pool) => { sqlClientPool, Sql.deleteDeployTargetConfigs(id) ); + // logger.debug(`deleting project ${id} environment leftover backups rows`) + // clean up backups table so backups for environments don't remain in limbo once the project is deleted + // @INFO: this step is to clear out any backup rows from the database for a project that is being deleted + // at the moment this only happens when the project is deleted + // but there is a commented section in the environment helpers where when an environment + // is deleted, the deletion of the data for the environment could handle this step instead + const projectEnvironmentIds = await query(sqlClientPool, environmentSql.selectEnvironmentsByProjectID(id, true)); + await query( + sqlClientPool, + backupSql.deleteBackupsByEnvironmentIds(projectEnvironmentIds) + ); + // same for environment storage + // logger.debug(`deleting project ${id} environment leftover storage rows`) + await query( + sqlClientPool, + environmentSql.deleteEnvironmentStorageByEnvironmentIds(projectEnvironmentIds) + ); // logger.debug(`deleting project ${id} leftover environment rows`) // clean up environments table so environments don't remain in limbo once the project is deleted await query(