Bugfixes storage postgres interface and added repo integration tests

.github/workflows/build-and-analyze.yml

@@ -20,6 +20,20 @@ jobs:
     name: Build, test & analyze
     if: ((github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) || github.event_name == 'push') &&  github.repository_owner == 'Altinn' && github.actor != 'dependabot[bot]'
     runs-on: ubuntu-latest
+    services:
+     postgres:
+       image: postgres:15
+       env:
+         POSTGRES_USER: platform_storage_admin
+         POSTGRES_PASSWORD: Password
+         POSTGRES_DB: storagedb
+       options: >-
+         --health-cmd pg_isready
+         --health-interval 10s
+         --health-timeout 5s
+         --health-retries 5
+       ports:
+         - 5432:5432
     steps:
       - name: Setup .NET
         uses: actions/setup-dotnet@v3
@@ -35,6 +49,10 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+      - name: Setup PostgreSQL
+        run: |
+            chmod +x dbsetup.sh
+            ./dbsetup.sh
       - name: Cache SonarCloud packages
         uses: actions/cache@v3
         with:
@@ -49,7 +67,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         run: |
-          dotnet-sonarscanner begin /k:"Altinn_altinn-storage" /o:"altinn" /d:sonar.login="${{ secrets.SONAR_TOKEN }}" /d:sonar.host.url="https://sonarcloud.io" /d:sonar.cs.vstest.reportsPaths="**/*.trx" /d:sonar.cs.opencover.reportsPaths="**/coverage.opencover.xml" /d:sonar.cpd.exclusions="**/Swagger/*Filter.cs,src/Storage/Migration/**/*.sql" /d:sonar.coverage.exclusions="src/Storage/Migration/"
+          dotnet-sonarscanner begin /k:"Altinn_altinn-storage" /o:"altinn" /d:sonar.login="${{ secrets.SONAR_TOKEN }}" /d:sonar.host.url="https://sonarcloud.io" /d:sonar.cs.vstest.reportsPaths="**/*.trx" /d:sonar.cs.opencover.reportsPaths="**/coverage.opencover.xml" /d:sonar.exclusions="src/Storage/Migration/**/*"
 
           dotnet build Altinn.Platform.Storage.sln -v q
 

dbsetup.sh

@@ -0,0 +1,9 @@
+ #!/bin/bash
+export PGPASSWORD=Password
+
+# set up platform_storage role
+psql -h localhost -p 5432 -U platform_storage_admin -d storagedb \
+-c "DO \$\$
+    BEGIN CREATE ROLE platform_storage WITH LOGIN  PASSWORD 'Password';
+    EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE;
+    END \$\$;"

src/DataCleanup/NightlyCleanup.cs

@@ -45,10 +45,10 @@
         {
             List<Instance> instances = await _cosmosService.GetHardDeletedInstances();
             List<Application> applications = await _cosmosService.GetApplications(instances.Select(i => i.AppId).Distinct().ToList());
             List<string> autoDeleteAppIds = applications.Where(a => a.AutoDeleteOnProcessEnd == true).Select(a => a.Id).ToList();
             int successfullyDeleted = 0;
 
-            Stopwatch stopwatch = new Stopwatch();
+            Stopwatch stopwatch = new();
             stopwatch.Start();
 
             foreach (Instance instance in instances)

src/Storage/Controllers/CleanupController.cs

@@ -0,0 +1,245 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading.Tasks;
+using Altinn.Platform.Storage.Configuration;
+using Altinn.Platform.Storage.Interface.Models;
+using Altinn.Platform.Storage.Repository;
+
+using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Primitives;
+
+namespace Altinn.Platform.Storage.Controllers
+{
+    /// <summary>
+    /// Handles cleanup of storage data
+    /// </summary>
+    [Route("storage/api/v1/cleanup")]
+    [ApiController]
+    public class CleanupController : ControllerBase
+    {
+        private readonly IInstanceRepository _instanceRepository;
+        private readonly IApplicationRepository _applicationRepository;
+        private readonly IBlobRepository _blobRepository;
+        private readonly IDataRepository _dataRepository;
+        private readonly IInstanceEventRepository _instanceEventRepository;
+        private readonly ILogger<CleanupController> _logger;
+        private readonly bool _usePostgreSQL;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="CleanupController"/> class
+        /// </summary>
+        /// <param name="instanceRepository">the instance repository handler</param>
+        /// <param name="applicationRepository">the application repository handler</param>
+        /// <param name="blobRepository">the blob repository handler</param>
+        /// <param name="dataRepository">the data repository handler</param>
+        /// <param name="instanceEventRepository">the instance event repository handler</param>
+        /// <param name="logger">the logger</param>
+        /// <param name="generalSettings">the general settings.</param>
+        public CleanupController(
+            IInstanceRepository instanceRepository,
+            IApplicationRepository applicationRepository,
+            IBlobRepository blobRepository,
+            IDataRepository dataRepository,
+            IInstanceEventRepository instanceEventRepository,
+            ILogger<CleanupController> logger,
+            IOptions<GeneralSettings> generalSettings)
+        {
+            _instanceRepository = instanceRepository;
+            _applicationRepository = applicationRepository;
+            _blobRepository = blobRepository;
+            _dataRepository = dataRepository;
+            _instanceEventRepository = instanceEventRepository;
+            _logger = logger;
+            _usePostgreSQL = generalSettings.Value.UsePostgreSQL;
+        }
+
+        /// <summary>
+        /// Invoke periodic cleanup of instances
+        /// </summary>
+        /// <returns>?</returns>
+        [HttpDelete("cleanupinstances")]
+        [ProducesResponseType(StatusCodes.Status200OK)]
+        [ApiExplorerSettings(IgnoreApi = true)]
+        public async Task<ActionResult> CleanupInstances()
+        {
+            if (!_usePostgreSQL)
+            {
+                return Ok();
+            }
+
+            List<Instance> instances = await _instanceRepository.GetHardDeletedInstances();
+            List<string> autoDeleteAppIds = (await _applicationRepository.FindAll())
+                .Where(a => instances.Select(i => i.AppId).ToList().Contains(a.Id) && a.AutoDeleteOnProcessEnd)
+                .Select(a => a.Id).ToList();
+
+            Stopwatch stopwatch = Stopwatch.StartNew();
+            int successfullyDeleted = await CleanupInstancesInternal(instances, autoDeleteAppIds, "NightlyCleanup");
+            stopwatch.Stop();
+
+            _logger.LogInformation(
+                "CleanupController// CleanupInstances // {DeleteCount} of {OriginalCount} instances deleted in {Duration} s",
+                successfullyDeleted,
+                instances.Count,
+                stopwatch.Elapsed.TotalSeconds);
+
+            return Ok();
+        }
+
+        /// <summary>
+        /// Invoke periodic cleanup of instances for a specific app
+        /// </summary>
+        /// <returns>?</returns>
+        [HttpDelete("cleanupinstancesforapp/{appId}")]
+        [ProducesResponseType(StatusCodes.Status200OK)]
+        [ApiExplorerSettings(IgnoreApi = true)]
+        public async Task<ActionResult> CleanupInstancesForApp(string appId)
+        {
+            if (!_usePostgreSQL)
+            {
+                return Ok();
+            }
+
+            int successfullyDeleted = 0;
+            int processed = 0;
+            InstanceQueryResponse instancesResponse = new() { ContinuationToken = null };
+            Dictionary<string, StringValues> options = new() { { "appId", appId } };
+
+            Stopwatch stopwatch = Stopwatch.StartNew();
+            do
+            {
+                instancesResponse = await _instanceRepository.GetInstancesFromQuery(options, instancesResponse.ContinuationToken, 5000);
+                successfullyDeleted += await CleanupInstancesInternal(instancesResponse.Instances, new List<string>(), nameof(CleanupInstancesForApp));
+                processed += (int)instancesResponse.Count;
+            }
+            while (instancesResponse.ContinuationToken != null);
+            stopwatch.Stop();
+
+            _logger.LogInformation(
+                "CleanupController // CleanupInstancesForApp // {DeleteCount} of {OriginalCount} instances deleted in {Duration} s",
+                successfullyDeleted,
+                processed,
+                stopwatch.Elapsed.TotalSeconds);
+
+            return Ok();
+        }
+
+        /// <summary>
+        /// Invoke periodic cleanup of data elements
+        /// </summary>
+        /// <returns>?</returns>
+        [HttpDelete("cleanupdataelements")]
+        [ProducesResponseType(StatusCodes.Status200OK)]
+        [Produces("application/json")]
+        [ApiExplorerSettings(IgnoreApi = true)]
+        public async Task<ActionResult> CleanupDataelements()
+        {
+            if (!_usePostgreSQL)
+            {
+                return Ok();
+            }
+
+            List<DataElement> dataElements = await _instanceRepository.GetHardDeletedDataElements();
+
+            int successfullyDeleted = 0;
+
+            Stopwatch stopwatch = Stopwatch.StartNew();
+
+            foreach (DataElement dataElement in dataElements)
+            {
+                bool dataBlobDeleted = false;
+
+                try
+                {
+                    dataBlobDeleted = await _blobRepository.DeleteBlob(dataElement.BlobStoragePath.Split('/')[0], dataElement.BlobStoragePath);
+                    if (dataBlobDeleted && await _dataRepository.Delete(dataElement))
+                    {
+                        successfullyDeleted++;
+                    }
+                }
+                catch (Exception e)
+                {
+                    _logger.LogError(
+                        e,
+                        "CleanupController // CleanupDataelements // Error occured when deleting dataElement Id: {dataElement.Id} Blobstoragepath: {blobStoragePath}",
+                        dataElement.Id,
+                        dataElement.BlobStoragePath);
+                }
+            }
+
+            stopwatch.Stop();
+            _logger.LogInformation(
+                "CleanupController // CleanupDataelements // {successfullyDeleted} of {count} data elements deleted in {totalSeconds} s",
+                successfullyDeleted,
+                dataElements.Count,
+                stopwatch.Elapsed.TotalSeconds);
+
+            return Ok();
+        }
+
+        private async Task<int> CleanupInstancesInternal(List<Instance> instances, List<string> autoDeleteAppIds, string caller)
+        {
+            int successfullyDeleted = 0;
+            foreach (Instance instance in instances)
+            {
+                bool blobsNoException = false;
+                bool instanceEventsNoException = false;
+                bool dataElementsNoException = false;
+
+                try
+                {
+                    blobsNoException = await _blobRepository.DeleteDataBlobs(instance);
+
+                    if (blobsNoException)
+                    {
+                        dataElementsNoException = await _dataRepository.DeleteForInstance(instance.Id.Split('/')[^1]);
+                    }
+
+                    try
+                    {
+                        if (autoDeleteAppIds.Contains(instance.AppId))
+                        {
+                            await _instanceEventRepository.DeleteAllInstanceEvents(instance.Id);
+                            instanceEventsNoException = true;
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogError(
+                            "{Caller} error deleting instance events for id {id}, {message}",
+                            caller,
+                            instance.Id,
+                            ex.Message);
+                    }
+
+                    if (dataElementsNoException
+                        && (!autoDeleteAppIds.Contains(instance.AppId) || instanceEventsNoException))
+                    {
+                        await _instanceRepository.Delete(instance);
+                        successfullyDeleted += 1;
+                        _logger.LogInformation(
+                            "{Caller} // Run // Instance deleted: {AppId}/{InstanceId}",
+                            caller,
+                            instance.AppId,
+                            $"{instance.InstanceOwner.PartyId}/{instance.Id}");
+                    }
+                }
+                catch (Exception e)
+                {
+                    _logger.LogError(
+                        e,
+                        "{Caller} // Run // Error occured when deleting instance: {AppId}/{InstanceId}",
+                        caller,
+                        instance.AppId,
+                        $"{instance.InstanceOwner.PartyId}/{instance.Id}");
+                }
+            }
+
+            return successfullyDeleted;
+        }
+    }
+}