From cd6d1e95ee872b5b97b9aa7547664dd5cded636a Mon Sep 17 00:00:00 2001 From: Yong Zhi Yuan Date: Sun, 28 Nov 2021 10:46:08 +0800 Subject: [PATCH] Decouple Change from Image --- .../workflows/azure-messaging-function.yml | 37 +++++++++++++++++ Api/Data/ChangeContext.cs | 4 ++ Api/Features/Change/DeleteChangeFunction.cs | 35 ++++------------ .../Change/SynchronizeChangeFunction.cs | 1 + Api/Features/Event/CreateEventFunction.cs | 1 + Api/Features/Event/DeleteEventFunction.cs | 1 + Api/Features/Event/UpdateEventFunction.cs | 1 + Api/Features/Image/CreateImageFunction.cs | 1 + Api/Features/Image/DeleteImageFunction.cs | 1 + Api/Features/Image/UpdateImageFunction.cs | 1 + Api/Features/Issue/CompleteTaskFunction.cs | 1 + Api/Features/Issue/CreateIssueFunction.cs | 1 + Api/Features/Issue/DeleteIssueFunction.cs | 1 + Api/Features/Issue/UpdateIssueFunction.cs | 1 + Api/Model/Change.cs | 12 +++++- Couple.sln | 6 +++ Messaging/Couple.Messaging.csproj | 21 ++++++++++ .../Features/ChangeDeletedEventFunction.cs | 40 +++++++++++++++++++ .../Features/ImageDeletedEventFunction.cs | 20 ++++++++++ Messaging/Model/Change.cs | 18 +++++++++ Messaging/Model/Event.cs | 14 +++++++ Messaging/Properties/launchSettings.json | 9 +++++ Messaging/host.json | 10 +++++ Messaging/local.settings.json | 16 ++++++++ 24 files changed, 225 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/azure-messaging-function.yml create mode 100644 Messaging/Couple.Messaging.csproj create mode 100644 Messaging/Features/ChangeDeletedEventFunction.cs create mode 100644 Messaging/Features/ImageDeletedEventFunction.cs create mode 100644 Messaging/Model/Change.cs create mode 100644 Messaging/Model/Event.cs create mode 100644 Messaging/Properties/launchSettings.json create mode 100644 Messaging/host.json create mode 100644 Messaging/local.settings.json diff --git a/.github/workflows/azure-messaging-function.yml b/.github/workflows/azure-messaging-function.yml new file mode 100644 index 00000000..8cb456a4 --- /dev/null +++ b/.github/workflows/azure-messaging-function.yml @@ -0,0 +1,37 @@ +# https://docs.microsoft.com/en-us/azure/azure-functions/functions-how-to-github-actions?tabs=dotnet#deploy-the-function-app + +name: Deploy DotNet project to function app with a Linux environment + +on: + [push] + +env: + AZURE_FUNCTIONAPP_NAME: change-event + AZURE_FUNCTIONAPP_PACKAGE_PATH: '.\Messaging' + DOTNET_VERSION: '6.0.x' + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + steps: + - name: 'Checkout GitHub Action' + uses: actions/checkout@v2 + + - name: Setup DotNet ${{ env.DOTNET_VERSION }} Environment + uses: actions/setup-dotnet@v1 + with: + dotnet-version: ${{ env.DOTNET_VERSION }} + + - name: 'Resolve Project Dependencies Using Dotnet' + shell: bash + run: | + pushd './${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}' + dotnet build --configuration Release --output ./output + popd + - name: 'Run Azure Functions Action' + uses: Azure/functions-action@v1 + id: fa + with: + app-name: ${{ env.AZURE_FUNCTIONAPP_NAME }} + package: '${{ env.AZURE_FUNCTIONAPP_PACKAGE_PATH }}/output' + publish-profile: ${{ secrets.AZURE_MESSAGING_FUNCTIONAPP_PUBLISH_PROFILE }} diff --git a/Api/Data/ChangeContext.cs b/Api/Data/ChangeContext.cs index 2b360086..f051b1f8 100644 --- a/Api/Data/ChangeContext.cs +++ b/Api/Data/ChangeContext.cs @@ -21,6 +21,10 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .Property(change => change.Id) .ToJsonProperty("id") .HasConversion(); + + modelBuilder.Entity() + .Property(change => change.Ttl) + .ToJsonProperty("ttl"); } } } diff --git a/Api/Features/Change/DeleteChangeFunction.cs b/Api/Features/Change/DeleteChangeFunction.cs index e892f848..51adcbb4 100644 --- a/Api/Features/Change/DeleteChangeFunction.cs +++ b/Api/Features/Change/DeleteChangeFunction.cs @@ -1,12 +1,8 @@ -using System; using System.Linq; using System.Net; -using System.Text.Json; using System.Threading.Tasks; -using Azure.Storage.Blobs; using Couple.Api.Data; using Couple.Api.Infrastructure; -using Couple.Shared.Model; using Couple.Shared.Model.Change; using FluentValidation; using Microsoft.Azure.Functions.Worker; @@ -22,7 +18,7 @@ public class DeleteChangesFunction private readonly ChangeContext _context; public DeleteChangesFunction(ICurrentUserService currentUserService, - ChangeContext context) + ChangeContext context) { _currentUserService = currentUserService; _context = context; @@ -53,13 +49,6 @@ public async Task Run( .Where(change => model.Guids.Contains(change.Id)) .ToListAsync(); - var areIdsValid = model.Guids.Count == toDelete.Count; - - if (!areIdsValid) - { - return req.CreateResponse(HttpStatusCode.BadRequest); - } - var claims = _currentUserService.GetClaims(req.Headers); var canDelete = toDelete.All(change => change.UserId == claims.Id); @@ -68,24 +57,16 @@ public async Task Run( return req.CreateResponse(HttpStatusCode.Forbidden); } + foreach (var change in toDelete) + { + change.Ttl = 3600; + } + _context .Changes - .RemoveRange(toDelete); - await _context.SaveChangesAsync(); - - var imageIdsToDelete = toDelete - .Where(change => change.Command == Command.CreateImage || change.Command == Command.UpdateImage) - .Select(change => change.Content) - .Select(content => JsonSerializer.Deserialize(content)) - .Select(image => image.Id.ToString()) - .ToList(); + .UpdateRange(toDelete); - var connectionString = Environment.GetEnvironmentVariable("ImagesConnectionString"); - foreach (var id in imageIdsToDelete) - { - var client = new BlobClient(connectionString, "images", id); - await client.DeleteIfExistsAsync(); - } + await _context.SaveChangesAsync(); return req.CreateResponse(HttpStatusCode.OK); } diff --git a/Api/Features/Change/SynchronizeChangeFunction.cs b/Api/Features/Change/SynchronizeChangeFunction.cs index 74f0fbcb..3b0f8b74 100644 --- a/Api/Features/Change/SynchronizeChangeFunction.cs +++ b/Api/Features/Change/SynchronizeChangeFunction.cs @@ -46,6 +46,7 @@ public async Task Run( var toReturn = await _context .Changes .Where(change => change.UserId == claims.Id) + .Where(change => change.Ttl == -1) .OrderBy(change => change.Timestamp) .ProjectTo(_mapper.ConfigurationProvider) .ToListAsync(); diff --git a/Api/Features/Event/CreateEventFunction.cs b/Api/Features/Event/CreateEventFunction.cs index f0232914..51788f52 100644 --- a/Api/Features/Event/CreateEventFunction.cs +++ b/Api/Features/Event/CreateEventFunction.cs @@ -56,6 +56,7 @@ public async Task Run( Command.CreateEvent, claims.PartnerId, _dateTimeService.Now, + form.Value.Event.Id, form.Json); _context diff --git a/Api/Features/Event/DeleteEventFunction.cs b/Api/Features/Event/DeleteEventFunction.cs index 26c7b567..d95d742c 100644 --- a/Api/Features/Event/DeleteEventFunction.cs +++ b/Api/Features/Event/DeleteEventFunction.cs @@ -41,6 +41,7 @@ public async Task Run( Command.DeleteEvent, claims.PartnerId, _dateTimeService.Now, + id, JsonSerializer.Serialize(id)); _context diff --git a/Api/Features/Event/UpdateEventFunction.cs b/Api/Features/Event/UpdateEventFunction.cs index 8de27b17..870ee3b3 100644 --- a/Api/Features/Event/UpdateEventFunction.cs +++ b/Api/Features/Event/UpdateEventFunction.cs @@ -56,6 +56,7 @@ public async Task Run( Command.UpdateEvent, claims.PartnerId, _dateTimeService.Now, + form.Value.Event.Id, form.Json); _changeContext diff --git a/Api/Features/Image/CreateImageFunction.cs b/Api/Features/Image/CreateImageFunction.cs index c0746db8..f9aba1ca 100644 --- a/Api/Features/Image/CreateImageFunction.cs +++ b/Api/Features/Image/CreateImageFunction.cs @@ -65,6 +65,7 @@ public async Task Run( Command.CreateImage, claims.PartnerId, _dateTimeService.Now, + dto.Id, JsonSerializer.Serialize(_mapper.Map(dto))); _context diff --git a/Api/Features/Image/DeleteImageFunction.cs b/Api/Features/Image/DeleteImageFunction.cs index d57d46c1..c07dd20c 100644 --- a/Api/Features/Image/DeleteImageFunction.cs +++ b/Api/Features/Image/DeleteImageFunction.cs @@ -41,6 +41,7 @@ public async Task Run( Command.DeleteImage, claims.PartnerId, _dateTimeService.Now, + id, JsonSerializer.Serialize(id)); _context diff --git a/Api/Features/Image/UpdateImageFunction.cs b/Api/Features/Image/UpdateImageFunction.cs index 0901f1bb..7034db8a 100644 --- a/Api/Features/Image/UpdateImageFunction.cs +++ b/Api/Features/Image/UpdateImageFunction.cs @@ -64,6 +64,7 @@ public async Task Run( Command.UpdateImage, claims.PartnerId, _dateTimeService.Now, + dto.Id, JsonSerializer.Serialize(_mapper.Map(dto))); _context diff --git a/Api/Features/Issue/CompleteTaskFunction.cs b/Api/Features/Issue/CompleteTaskFunction.cs index 7da61b39..64aa0ecf 100644 --- a/Api/Features/Issue/CompleteTaskFunction.cs +++ b/Api/Features/Issue/CompleteTaskFunction.cs @@ -55,6 +55,7 @@ public async Task Run( Command.CompleteTask, claims.PartnerId, _dateTimeService.Now, + form.Value.Id, form.Json); _context diff --git a/Api/Features/Issue/CreateIssueFunction.cs b/Api/Features/Issue/CreateIssueFunction.cs index 15585fbc..ca90a53b 100644 --- a/Api/Features/Issue/CreateIssueFunction.cs +++ b/Api/Features/Issue/CreateIssueFunction.cs @@ -55,6 +55,7 @@ public async Task Run( Command.CreateIssue, claims.PartnerId, _dateTimeService.Now, + form.Value.Id, form.Json); _context diff --git a/Api/Features/Issue/DeleteIssueFunction.cs b/Api/Features/Issue/DeleteIssueFunction.cs index f6ab7698..7dad822c 100644 --- a/Api/Features/Issue/DeleteIssueFunction.cs +++ b/Api/Features/Issue/DeleteIssueFunction.cs @@ -41,6 +41,7 @@ public async Task Run( Command.DeleteIssue, claims.PartnerId, _dateTimeService.Now, + id, JsonSerializer.Serialize(id)); _context diff --git a/Api/Features/Issue/UpdateIssueFunction.cs b/Api/Features/Issue/UpdateIssueFunction.cs index cc1c7772..f3d6291d 100644 --- a/Api/Features/Issue/UpdateIssueFunction.cs +++ b/Api/Features/Issue/UpdateIssueFunction.cs @@ -55,6 +55,7 @@ public async Task Run( Command.UpdateIssue, claims.PartnerId, _dateTimeService.Now, + form.Value.Id, form.Json); _context diff --git a/Api/Model/Change.cs b/Api/Model/Change.cs index 43f92ee9..8676d15f 100644 --- a/Api/Model/Change.cs +++ b/Api/Model/Change.cs @@ -8,17 +8,27 @@ public class Change public string Command { get; init; } public string UserId { get; init; } public DateTime Timestamp { get; init; } + public Guid ContentId { get; init; } public string Content { get; init; } + public int? Ttl { get; set; } private Change() { } - public Change(Guid id, string command, string userId, DateTime timestamp, string content) + public Change(Guid id, string command, string userId, DateTime timestamp, Guid contentId, string content) { Id = id; Command = command; UserId = userId; Timestamp = timestamp; + ContentId = contentId; Content = content; + + // 1. Annotating this property with Json Attributes don't seem to work e.g. + // [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] and [JsonPropertyName("ttl")] + // 2. The alternative is to implement the converter + // https://docs.microsoft.com/en-us/dotnet/standard/serialization/system-text-json-migrate-from-newtonsoft-how-to?pivots=dotnet-6-0#conditionally-ignore-a-property + // which is much more troublesome than this solution. + Ttl = -1; } } } diff --git a/Couple.sln b/Couple.sln index 59caa0c6..522bacb8 100644 --- a/Couple.sln +++ b/Couple.sln @@ -9,6 +9,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Couple.Api", "Api\Couple.Ap EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Couple.Shared", "Shared\Couple.Shared.csproj", "{BF40FDE1-00B3-420B-B3C8-3AC60A075A85}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Couple.Messaging", "Messaging\Couple.Messaging.csproj", "{B6CF82EB-15D6-4E71-BEAE-F52FC2190109}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -27,6 +29,10 @@ Global {BF40FDE1-00B3-420B-B3C8-3AC60A075A85}.Debug|Any CPU.Build.0 = Debug|Any CPU {BF40FDE1-00B3-420B-B3C8-3AC60A075A85}.Release|Any CPU.ActiveCfg = Release|Any CPU {BF40FDE1-00B3-420B-B3C8-3AC60A075A85}.Release|Any CPU.Build.0 = Release|Any CPU + {B6CF82EB-15D6-4E71-BEAE-F52FC2190109}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B6CF82EB-15D6-4E71-BEAE-F52FC2190109}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B6CF82EB-15D6-4E71-BEAE-F52FC2190109}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B6CF82EB-15D6-4E71-BEAE-F52FC2190109}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Messaging/Couple.Messaging.csproj b/Messaging/Couple.Messaging.csproj new file mode 100644 index 00000000..61acdd19 --- /dev/null +++ b/Messaging/Couple.Messaging.csproj @@ -0,0 +1,21 @@ + + + net6.0 + v4 + latest + enable + + + + + + + + + + + + + + + diff --git a/Messaging/Features/ChangeDeletedEventFunction.cs b/Messaging/Features/ChangeDeletedEventFunction.cs new file mode 100644 index 00000000..6b2fe884 --- /dev/null +++ b/Messaging/Features/ChangeDeletedEventFunction.cs @@ -0,0 +1,40 @@ +using Azure.Messaging.EventGrid; +using Couple.Messaging.Model; +using Couple.Shared.Model; +using Microsoft.Azure.Documents; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.EventGrid; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; + +namespace Couple.Messaging.Features +{ + public class DeletedEventFunction + { + [FunctionName("ChangeDeletedEventFunction")] + public async Task Run([CosmosDBTrigger("%DatabaseName%", + "%CollectionName%", + ConnectionStringSetting = "DatabaseConnectionString", + CreateLeaseCollectionIfNotExists = true)] IReadOnlyList documents, + [EventGrid(TopicEndpointUri = "EventGridEndpoint", + TopicKeySetting = "EventGridKey")] IAsyncCollector eventCollector) + { + var changes = documents.Select(d => d.ToString()) + .Select(json => JsonSerializer.Deserialize(json)!) + .Where(change => change.Ttl != -1) + .Where(change => change.Command is Command.CreateImage or Command.UpdateImage) + .ToList(); + + List tasks = new(); + foreach (var change in changes) + { + var task = eventCollector.AddAsync(new(change.ContentId.ToString(), "ImageDeleted", "1", new { })); + tasks.Add(task); + } + + await Task.WhenAll(tasks); + } + } +} diff --git a/Messaging/Features/ImageDeletedEventFunction.cs b/Messaging/Features/ImageDeletedEventFunction.cs new file mode 100644 index 00000000..0efb7dfa --- /dev/null +++ b/Messaging/Features/ImageDeletedEventFunction.cs @@ -0,0 +1,20 @@ +using Azure.Storage.Blobs; +using Couple.Messaging.Model; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.EventGrid; +using System; +using System.Threading.Tasks; + +namespace Couple.Messaging.Features +{ + public class ImageDeletedEventFunction + { + [FunctionName("ImageDeletedEventFunction")] + public async Task Run([EventGridTrigger] Event @event) + { + var connectionString = Environment.GetEnvironmentVariable("ImagesConnectionString"); + var client = new BlobClient(connectionString, "images", @event.Subject); + await client.DeleteIfExistsAsync(); + } + } +} diff --git a/Messaging/Model/Change.cs b/Messaging/Model/Change.cs new file mode 100644 index 00000000..459d52fd --- /dev/null +++ b/Messaging/Model/Change.cs @@ -0,0 +1,18 @@ +using System; +using System.Text.Json.Serialization; + +namespace Couple.Messaging.Model +{ + public class Change + { + public Guid Id { get; init; } + public string? Command { get; init; } + public string? UserId { get; init; } + public DateTime Timestamp { get; init; } + public Guid ContentId { get; init; } + public string? Content { get; init; } + + [JsonPropertyName("ttl")] + public int? Ttl { get; set; } + } +} diff --git a/Messaging/Model/Event.cs b/Messaging/Model/Event.cs new file mode 100644 index 00000000..97c5ac37 --- /dev/null +++ b/Messaging/Model/Event.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; + +namespace Couple.Messaging.Model +{ + public class Event + { + public string? Topic { get; init; } + public string? Subject { get; init; } + public string? EventType { get; init; } + public DateTime EventTime { get; init; } + public IDictionary? Data { get; init; } + } +} diff --git a/Messaging/Properties/launchSettings.json b/Messaging/Properties/launchSettings.json new file mode 100644 index 00000000..fc1bf438 --- /dev/null +++ b/Messaging/Properties/launchSettings.json @@ -0,0 +1,9 @@ +{ + "profiles": { + "Api": { + "commandName": "Executable", + "executablePath": "func", + "commandLineArgs": "start" + } + } +} diff --git a/Messaging/host.json b/Messaging/host.json new file mode 100644 index 00000000..898c638e --- /dev/null +++ b/Messaging/host.json @@ -0,0 +1,10 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true + } + } + } +} \ No newline at end of file diff --git a/Messaging/local.settings.json b/Messaging/local.settings.json new file mode 100644 index 00000000..f7b474b8 --- /dev/null +++ b/Messaging/local.settings.json @@ -0,0 +1,16 @@ +{ + "IsEncrypted": false, + "Values": { + "FUNCTIONS_WORKER_RUNTIME": "dotnet", + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "CollectionName": "change", + "DatabaseName": "database", + "DatabaseConnectionString": "", + "EventGridEndpoint": "", + "EventGridKey": "", + "ImagesConnectionString": "" + }, + "Host": { + "CORS": "*" + } +}