bug/met 6244 add vacuum procedure to repositories (#170)

* MET-6244 add vacuum cleaner * MET-6244 fixing removal order for records tables * MET-6244 fix table names
europeana · Nov 13, 2024 · 3c538f1 · 3c538f1
1 parent bc50bb4
commit 3c538f1
Show file tree

Hide file tree

Showing 4 changed files with 120 additions and 3 deletions.
diff --git a/src/main/java/eu/europeana/metis/sandbox/service/dataset/DatasetRemoverServiceImpl.java b/src/main/java/eu/europeana/metis/sandbox/service/dataset/DatasetRemoverServiceImpl.java
@@ -6,12 +6,16 @@
 import eu.europeana.metis.sandbox.service.record.RecordLogService;
 import eu.europeana.metis.sandbox.service.record.RecordService;
 import eu.europeana.metis.sandbox.service.util.ThumbnailStoreService;
+import eu.europeana.metis.sandbox.service.util.VacuumService;
 import eu.europeana.metis.sandbox.service.workflow.IndexingService;
 import java.util.List;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 
+/**
+ * The type Dataset remover service.
+ */
 @Service
 class DatasetRemoverServiceImpl implements DatasetRemoverService {
 
@@ -26,6 +30,22 @@ class DatasetRemoverServiceImpl implements DatasetRemoverService {
   private final ProblemPatternDataRemover problemPatternDataRemover;
   private final HarvestingParameterService harvestingParameterService;
   private final DeBiasStateService debiasStateService;
+  private final VacuumService vacuumService;
+
+  /**
+   * Instantiates a new Dataset remover service.
+   *
+   * @param datasetService the dataset service
+   * @param datasetLogService the dataset log service
+   * @param recordLogService the record log service
+   * @param indexingService the indexing service
+   * @param thumbnailStoreService the thumbnail store service
+   * @param recordService the record service
+   * @param problemPatternDataRemover the problem pattern data remover
+   * @param harvestingParameterService the harvesting parameter service
+   * @param debiasStateService the debias state service
+   * @param vacuumService the vacuum service
+   */
   DatasetRemoverServiceImpl(
           DatasetService datasetService,
           DatasetLogService datasetLogService,
@@ -35,7 +55,8 @@ class DatasetRemoverServiceImpl implements DatasetRemoverService {
           RecordService recordService,
           ProblemPatternDataRemover problemPatternDataRemover,
           HarvestingParameterService harvestingParameterService,
-          DeBiasStateService debiasStateService) {
+          DeBiasStateService debiasStateService,
+          VacuumService vacuumService) {
     this.datasetService = datasetService;
     this.datasetLogService = datasetLogService;
     this.recordLogService = recordLogService;
@@ -45,6 +66,7 @@ class DatasetRemoverServiceImpl implements DatasetRemoverService {
     this.problemPatternDataRemover = problemPatternDataRemover;
     this.harvestingParameterService = harvestingParameterService;
     this.debiasStateService = debiasStateService;
+    this.vacuumService = vacuumService;
   }
 
   @Override
@@ -66,6 +88,8 @@ public void remove(int days) {
           // remove from sandbox
           LOGGER.info("Remove record logs for dataset id: [{}]", dataset);
           recordLogService.remove(dataset);
+          LOGGER.info("Remove debias report with id: [{}]", dataset);
+          debiasStateService.cleanDeBiasReport(Integer.valueOf(dataset));
           LOGGER.info("Remove records for dataset id: [{}]", dataset);
           recordService.remove(dataset);
           LOGGER.info("Remove logs for dataset id: [{}]", dataset);
@@ -76,12 +100,12 @@ public void remove(int days) {
           problemPatternDataRemover.removeProblemPatternDataFromDatasetId(dataset);
           LOGGER.info("Remove dataset with id: [{}]", dataset);
           datasetService.remove(dataset);
-          LOGGER.info("Remove debias report with id: [{}]", dataset);
-          debiasStateService.cleanDeBiasReport(Integer.valueOf(dataset));
         } catch (ServiceException e) {
           LOGGER.error("Failed to remove dataset [{}] ", dataset, e);
         }
       });
+      LOGGER.info("Data removal completed, performing vacuum cleaning...");
+      vacuumService.vacuum();
     } catch (RuntimeException exception) {
       LOGGER.error("General failure to remove dataset", exception);
     }

diff --git a/src/main/java/eu/europeana/metis/sandbox/service/util/VacuumService.java b/src/main/java/eu/europeana/metis/sandbox/service/util/VacuumService.java
@@ -0,0 +1,56 @@
+package eu.europeana.metis.sandbox.service.util;
+
+import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Service;
+
+/**
+ * The type Vacuum service.
+ */
+@Service
+public class VacuumService {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(VacuumService.class);
+
+  private final JdbcTemplate jdbcTemplate;
+
+  /**
+   * Instantiates a new Vacuum service.
+   *
+   * @param jdbcTemplate the jdbc template
+   */
+  public VacuumService(JdbcTemplate jdbcTemplate) {
+    this.jdbcTemplate = jdbcTemplate;
+  }
+
+  /**
+   * Vacuum the listed tables.
+   */
+  public void vacuum() {
+    List<String> tables = List.of(
+        "problem_patterns.record_problem_pattern",
+        "problem_patterns.record_problem_pattern_occurrence",
+        "problem_patterns.record_title",
+        "problem_patterns.dataset_problem_pattern",
+        "problem_patterns.execution_point",
+        "public.record_log",
+        "public.record_error_log",
+        "public.record",
+        "public.harvesting_parameter",
+        "public.record_debias_detail",
+        "public.record_debias_main",
+        "public.dataset_debias_detect",
+        "public.dataset_log",
+        "public.dataset",
+        "integration.int_lock",
+        "rate_limit.buckets"
+    );
+    tables.forEach(table -> {
+      LOGGER.info("Performing vacuum of table {}", table);
+      jdbcTemplate.execute(String.format("VACUUM %s", table));
+      LOGGER.info("Vacuum of table {} completed", table);
+    });
+  }
+}
diff --git a/src/test/java/eu/europeana/metis/sandbox/service/dataset/DatasetRemoverServiceImplTest.java b/src/test/java/eu/europeana/metis/sandbox/service/dataset/DatasetRemoverServiceImplTest.java
@@ -13,6 +13,7 @@
 import eu.europeana.metis.sandbox.service.record.RecordLogService;
 import eu.europeana.metis.sandbox.service.record.RecordService;
 import eu.europeana.metis.sandbox.service.util.ThumbnailStoreService;
+import eu.europeana.metis.sandbox.service.util.VacuumService;
 import eu.europeana.metis.sandbox.service.workflow.IndexingService;
 import java.util.List;
 import org.junit.jupiter.api.Test;
@@ -51,6 +52,9 @@ class DatasetRemoverServiceImplTest {
   @Mock
   private DeBiasStateService deBiasStateService;
 
+  @Mock
+  private VacuumService vacuumService;
+
   @InjectMocks
   private DatasetRemoverServiceImpl service;
 
@@ -70,6 +74,7 @@ void remove_expectSuccess() {
     verify(recordService, times(4)).remove(anyString());
     verify(problemPatternDataRemover, times(4)).removeProblemPatternDataFromDatasetId(anyString());
     verify(deBiasStateService, times(4)).cleanDeBiasReport(anyInt());
+    verify(vacuumService, times(1)).vacuum();
   }
 
   @Test

diff --git a/src/test/java/eu/europeana/metis/sandbox/service/util/VacuumServiceTest.java b/src/test/java/eu/europeana/metis/sandbox/service/util/VacuumServiceTest.java
@@ -0,0 +1,32 @@
+package eu.europeana.metis.sandbox.service.util;
+
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.springframework.jdbc.core.JdbcTemplate;
+
+@ExtendWith(MockitoExtension.class)
+class VacuumServiceTest {
+
+  @Mock
+  JdbcTemplate jdbcTemplate;
+
+  @InjectMocks
+  VacuumService vacuumService;
+
+  @Test
+  void vacuum() {
+    doNothing().when(jdbcTemplate).execute(anyString());
+
+    vacuumService.vacuum();
+
+    verify(jdbcTemplate,times(16)).execute(anyString());
+  }
+}