chore: update delete environment and project helpers to clear more st…

…ale data when removed

services/api/src/resources/backup/sql.ts

@@ -13,6 +13,12 @@ export const Sql = {
     knex('env_vars')
       .where('project', projectId)
       .toString(),
+  selectBackupsByEnvironmentId: (environmentId: number) =>
+    knex('environment_backup')
+      .where('environment', '=', environmentId)
+      .orderBy('created', 'desc')
+      .orderBy('id', 'desc')
+      .toString(),
   insertBackup: ({
     id,
     environment,
@@ -123,5 +129,17 @@ export const Sql = {
         'environment.id'
       )
       .where('environment_backup.backup_id', backupId)
-      .toString()
+      .toString(),
+  // delete all environments backups from backup table that match environment id
+  deleteBackupsByEnvironmentId: (environmentId: number) =>
+    knex('environment_backup')
+      .where('environment', '=', environmentId)
+      .delete()
+      .toString(),
+  // delete all environments backups from backup table that match environment ids
+  deleteBackupsByEnvironmentIds: (environmentIds: number[]) =>
+    knex('environment_backup')
+      .whereIn('environment', environmentIds)
+      .delete()
+      .toString(),
 };

services/api/src/resources/environment/helpers.ts

@@ -5,6 +5,7 @@ import { query } from '../../util/db';
 import { Sql } from './sql';
 import { Sql as problemSql } from '../problem/sql';
 import { Sql as factSql } from '../fact/sql';
+// import { Sql as backupSql } from '../backup/sql';
 import { Helpers as projectHelpers } from '../project/helpers';
 import { HistoryRetentionEnforcer } from '../retentionpolicy/history';
 import { logger } from '../../loggers/logger';
@@ -34,6 +35,29 @@ export const Helpers = (sqlClientPool: Pool) => {
     deleteEnvironment: async (name: string, eid: number, pid: number) => {
       const environmentData = await Helpers(sqlClientPool).getEnvironmentById(eid);
       const projectData = await projectHelpers(sqlClientPool).getProjectById(pid);
+
+      // attempt to run any retention policy processes before the environment is deleted
+      try {
+        // export a dump of the project, environment data, and associated task and deployment history before the environment is deleted
+        await HistoryRetentionEnforcer().saveEnvironmentHistoryBeforeDeletion(projectData, environmentData)
+      } catch (e) {
+        logger.error(`error running save environment history: ${e}`)
+      }
+      // purge all history for this environment, including logs and files from s3
+      try {
+        // remove all deployments and associated files
+        await HistoryRetentionEnforcer().cleanupAllDeployments(projectData, environmentData)
+      } catch (e) {
+        logger.error(`error running deployment retention enforcer: ${e}`)
+      }
+      try {
+        // remove all tasks and associated files
+        await HistoryRetentionEnforcer().cleanupAllTasks(projectData, environmentData)
+      } catch (e) {
+        logger.error(`error running task retention enforcer: ${e}`)
+      }
+
+      // then proceed to purge related data
       try {
         // clean up environment variables
         // logger.debug(`deleting environment ${name}/id:${eid}/project:${pid} environment variables`)
@@ -67,30 +91,30 @@ export const Helpers = (sqlClientPool: Pool) => {
           problemSql.deleteProblemsForEnvironment(eid)
         );
 
-        // @TODO: environment_storage, environment_backup
+        // delete the environment backups rows
+        // logger.debug(`deleting environment ${name}/id:${eid}/project:${pid} environment backups`)
+        // @TODO: this could be done here, but it would mean that to recover all the backup ids of a deleted environment
+        // in the event that an environment is "accidentally deleted" it would require accessing the bucket
+        // to retrieve them from the saved history export JSON dump
+        // this is disabled for now, but when a project is deleted, all of the backups for any environments of that project
+        // will have the table cleaned out to keep the database leaner
+        // await query(
+        //   sqlClientPool,
+        //   backupSql.deleteBackupsByEnvironmentId(eid)
+        // );
+        // clean up storage data
+        // logger.debug(`deleting environment ${name}/id:${eid}/project:${pid} environment storage`)
+        // @TODO: this could be done here, but amazee.io might still use this data for environments that are deleted
+        // this is disabled for now, but when a project is deleted, all of the storages for any environments of that project
+        // will have the table cleaned out to keep the database leaner
+        // await query(
+        //   sqlClientPool,
+        //   Sql.deleteEnvironmentStorageByEnvironmentId(eid)
+        // );
       } catch (e) {
         logger.error(`error cleaning up linked environment tables: ${e}`)
       }
 
-      try {
-        // export a dump of the project, environment data, and associated task and deployment history before the environment is deleted
-        await HistoryRetentionEnforcer().saveEnvironmentHistoryBeforeDeletion(projectData, environmentData)
-      } catch (e) {
-        logger.error(`error running save environment history: ${e}`)
-      }
-      // purge all history for this environment, including logs and files from s3
-      try {
-        // remove all deployments and associated files
-        await HistoryRetentionEnforcer().cleanupAllDeployments(projectData, environmentData)
-      } catch (e) {
-        logger.error(`error running deployment retention enforcer: ${e}`)
-      }
-      try {
-        // remove all tasks and associated files
-        await HistoryRetentionEnforcer().cleanupAllTasks(projectData, environmentData)
-      } catch (e) {
-        logger.error(`error running task retention enforcer: ${e}`)
-      }
       // delete the environment
       // logger.debug(`deleting environment ${name}/id:${eid}/project:${pid}`)
       await query(

services/api/src/resources/environment/sql.ts

@@ -198,4 +198,14 @@ export const Sql = {
       .where('project', '=', projectId)
       .delete()
       .toString(),
+  deleteEnvironmentStorageByEnvironmentId: (id: number) =>
+    knex('environment_storage')
+      .where('environment', '=', id)
+      .delete()
+      .toString(),
+  deleteEnvironmentStorageByEnvironmentIds: (ids: number[]) =>
+    knex('environment_storage')
+      .whereIn('environment', ids)
+      .delete()
+      .toString(),
 };

services/api/src/resources/project/helpers.ts

@@ -4,6 +4,7 @@ import { asyncPipe } from '@lagoon/commons/dist/util/func';
 import { query } from '../../util/db';
 import { Sql } from './sql';
 import { Sql as environmentSql } from '../environment/sql';
+import { Sql as backupSql } from '../backup/sql';
 // import { logger } from '../../loggers/logger';
 
 export const Helpers = (sqlClientPool: Pool) => {
@@ -193,6 +194,23 @@ export const Helpers = (sqlClientPool: Pool) => {
         sqlClientPool,
         Sql.deleteDeployTargetConfigs(id)
       );
+      // logger.debug(`deleting project ${id} environment leftover backups rows`)
+      // clean up backups table so backups for environments don't remain in limbo once the project is deleted
+      // @INFO: this step is to clear out any backup rows from the database for a project that is being deleted
+      // at the moment this only happens when the project is deleted
+      // but there is a commented section in the environment helpers where when an environment
+      // is deleted, the deletion of the data for the environment could handle this step instead
+      const projectEnvironmentIds = await query(sqlClientPool, environmentSql.selectEnvironmentsByProjectID(id, true));
+      await query(
+        sqlClientPool,
+        backupSql.deleteBackupsByEnvironmentIds(projectEnvironmentIds)
+      );
+      // same for environment storage
+      // logger.debug(`deleting project ${id} environment leftover storage rows`)
+      await query(
+        sqlClientPool,
+        environmentSql.deleteEnvironmentStorageByEnvironmentIds(projectEnvironmentIds)
+      );
       // logger.debug(`deleting project ${id} leftover environment rows`)
       // clean up environments table so environments don't remain in limbo once the project is deleted
       await query(