From d4355d71feb9445c2e2a223081c6b588839c9bbc Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Fri, 1 Sep 2023 09:11:39 -0400 Subject: [PATCH 01/10] Removed cancellation token passing after line 99. Opted to explicitly pass CancellationToken.None rather than leaving default for consistency with other code and added clarity. Fixes Cancellation token - inconsistent SMT training state #62 --ECL --- .../Services/SmtTransferEngineBuildJob.cs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineBuildJob.cs index a639ebef5..1a727c7c8 100644 --- a/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineBuildJob.cs +++ b/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineBuildJob.cs @@ -97,13 +97,12 @@ CancellationToken cancellationToken await using (await rwLock.WriterLockAsync(cancellationToken: cancellationToken)) { cancellationToken.ThrowIfCancellationRequested(); - await smtModelTrainer.SaveAsync(cancellationToken); - await truecaseTrainer.SaveAsync(cancellationToken); - cancellationToken.ThrowIfCancellationRequested(); + await smtModelTrainer.SaveAsync(CancellationToken.None); + await truecaseTrainer.SaveAsync(CancellationToken.None); ITruecaser truecaser = await _truecaserFactory.CreateAsync(engineId); IReadOnlyList segmentPairs = await _trainSegmentPairs.GetAllAsync( p => p.TranslationEngineRef == engine!.Id, - cancellationToken + CancellationToken.None ); using ( IInteractiveTranslationModel smtModel = _smtModelFactory.Create( @@ -119,9 +118,8 @@ CancellationToken cancellationToken await smtModel.TrainSegmentAsync( segmentPair.Source, segmentPair.Target, - cancellationToken: cancellationToken + cancellationToken: CancellationToken.None ); - cancellationToken.ThrowIfCancellationRequested(); } } From 9086e5e8dae28751c7ecf478e9838a6fa014eebd Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Fri, 1 Sep 2023 09:55:49 -0400 Subject: [PATCH 02/10] Fixes https://github.com/sillsdev/serval/issues/100 -- ECL --- .github/workflows/ci-e2e.yml | 64 ++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/ci-e2e.yml diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml new file mode 100644 index 000000000..cd107444d --- /dev/null +++ b/.github/workflows/ci-e2e.yml @@ -0,0 +1,64 @@ +name: "CI Build: E2E tests" + +on: + push: + branches: ["run_e2etests_on_release_#100"] + release: + types: [published] + workflow_dispatch: + +jobs: + build: + name: Build + runs-on: ubuntu-latest + timeout-minutes: 15 + + env: + SERVAL_CLIENT_ID: ${{ secrets.SERVAL_CLIENT_ID }} + SERVAL_CLIENT_SECRET: ${{ secrets.SERVAL_CLIENT_SECRET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + ClearML_AccessKey: ${{ secrets.CLEARML_ACCESSKEY }} + ClearML_SecretKey: ${{ secrets.CLEARML_SECRETKEY }} + SERVAL_HOST_URL: http://localhost + SERVAL_AUTH_URL: https://sil-appbuilder.auth0.com + + steps: + - uses: actions/checkout@v3 + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + with: + dotnet-version: 6.0.x + + - name: Get Machine + run: dotnet build && cd .. && git clone https://github.com/sillsdev/serval.git && cd serval && dotnet build + + - name: Restore dotnet tools + run: dotnet tool restore + + - name: Restore dependencies + run: dotnet restore + + - name: Build + run: dotnet build --no-restore + + - name: Start containers + run: docker compose -f "docker-compose.yml" up -d && sleep 20 #allow time for mongo to start up properly + + - name: Debug network + run: docker ps -a && docker logs --since 10m serval_cntr && docker logs --since 10m echo_cntr && docker logs --since 10m machine-engine-cntr && docker logs --since 10m serval-mongo-1 && docker logs --since 10m machine-job-cntr + + - name: Pre-Test + run: sudo mkdir -p /var/lib/serval && sudo chmod 777 /var/lib/serval + + - name: Test + run: dotnet test --no-build --verbosity normal --filter "TestCategory!=slow&TestCategory=E2E" + + - name: Debug network (Post test) + if: ${{ failure() }} + run: docker ps -a && docker logs --since 10m serval_cntr && docker logs --since 10m echo_cntr && docker logs --since 10m machine-engine-cntr && docker logs --since 10m serval-mongo-1 && docker logs --since 10m machine-job-cntr + + - name: Stop containers + if: ${{ success() || failure() }} + run: docker compose down From 6183000013ab2f3ad29005f0fe7558a1b5901557 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Fri, 1 Sep 2023 10:03:53 -0400 Subject: [PATCH 03/10] Fixed directory issue --ECL --- .github/workflows/ci-e2e.yml | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml index cd107444d..fba5cea23 100644 --- a/.github/workflows/ci-e2e.yml +++ b/.github/workflows/ci-e2e.yml @@ -31,20 +31,11 @@ jobs: with: dotnet-version: 6.0.x - - name: Get Machine + - name: Get Serval run: dotnet build && cd .. && git clone https://github.com/sillsdev/serval.git && cd serval && dotnet build - - name: Restore dotnet tools - run: dotnet tool restore - - - name: Restore dependencies - run: dotnet restore - - - name: Build - run: dotnet build --no-restore - - name: Start containers - run: docker compose -f "docker-compose.yml" up -d && sleep 20 #allow time for mongo to start up properly + run: docker compose -f "../serval/docker-compose.yml" up -d && sleep 20 #allow time for mongo to start up properly - name: Debug network run: docker ps -a && docker logs --since 10m serval_cntr && docker logs --since 10m echo_cntr && docker logs --since 10m machine-engine-cntr && docker logs --since 10m serval-mongo-1 && docker logs --since 10m machine-job-cntr @@ -53,7 +44,7 @@ jobs: run: sudo mkdir -p /var/lib/serval && sudo chmod 777 /var/lib/serval - name: Test - run: dotnet test --no-build --verbosity normal --filter "TestCategory!=slow&TestCategory=E2E" + run: cd ../serval && dotnet test --no-build --verbosity normal --filter "TestCategory!=slow&TestCategory=E2E" - name: Debug network (Post test) if: ${{ failure() }} @@ -61,4 +52,4 @@ jobs: - name: Stop containers if: ${{ success() || failure() }} - run: docker compose down + run: docker compose -f "../serval/docker-compose.yml" down From 7a7cb8a489f7c38df61ba67aa4011708ec630932 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Wed, 6 Sep 2023 16:16:26 -0400 Subject: [PATCH 04/10] Retry with added secrets --ECL From c9384c86e4886efa63fa416e76571f0c88b7f0e6 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Mon, 11 Sep 2023 13:02:29 -0400 Subject: [PATCH 05/10] Empty commit to trigger workflow --ECL --- .history/.gitignore_20230911115750 | 51 +++ .history/.gitignore_20230911115828 | 52 +++ ...MachineBuilderExtensions_20230907084316.cs | 317 ++++++++++++++++ ...MachineBuilderExtensions_20230907133310.cs | 317 ++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907084316.cs | 340 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907095831.cs | 340 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907135725.cs | 343 ++++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907135737.cs | 342 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907135738.cs | 342 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907135739.cs | 342 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907140302.cs | 342 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907140611.cs | 342 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907140709.cs | 340 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907140719.cs | 340 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907140720.cs | 340 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907141110.cs | 340 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907141112.cs | 340 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907141905.cs | 340 +++++++++++++++++ .../ClearMLNmtEngineService_20230906141540.cs | 51 +++ .../ClearMLNmtEngineService_20230911085826.cs | 51 +++ .../ClearMLNmtEngineService_20230911085829.cs | 51 +++ .../Services/S3FileStorage_20230906141540.cs | 95 +++++ .../Services/S3FileStorage_20230907104119.cs | 95 +++++ .../Services/S3FileStorage_20230907130153.cs | 97 +++++ .../Services/S3FileStorage_20230907130157.cs | 97 +++++ .../Services/S3FileStorage_20230907132651.cs | 98 +++++ .../Services/S3FileStorage_20230907132652.cs | 98 +++++ .../Services/S3FileStorage_20230907171652.cs | 98 +++++ .../Services/S3FileStorage_20230907171653.cs | 98 +++++ .../Services/S3FileStorage_20230907171816.cs | 107 ++++++ .../Services/S3FileStorage_20230907171821.cs | 107 ++++++ .../Services/S3FileStorage_20230907171829.cs | 108 ++++++ .../Services/S3FileStorage_20230907171837.cs | 107 ++++++ .../Services/S3FileStorage_20230907171838.cs | 107 ++++++ .../Services/S3FileStorage_20230908125336.cs | 107 ++++++ .../Services/S3WriteStream_20230906141540.cs | 76 ++++ .../Services/S3WriteStream_20230907104130.cs | 76 ++++ .../Services/S3WriteStream_20230907104133.cs | 76 ++++ .../Services/S3WriteStream_20230907130227.cs | 77 ++++ .../Services/S3WriteStream_20230907130326.cs | 79 ++++ .../Services/S3WriteStream_20230907130328.cs | 78 ++++ .../Services/S3WriteStream_20230907130334.cs | 78 ++++ .../Services/S3WriteStream_20230907130427.cs | 87 +++++ .../Services/S3WriteStream_20230907130442.cs | 75 ++++ .../Services/S3WriteStream_20230907130507.cs | 108 ++++++ .../Services/S3WriteStream_20230907130523.cs | 108 ++++++ .../Services/S3WriteStream_20230907130903.cs | 106 ++++++ .../Services/S3WriteStream_20230907130942.cs | 107 ++++++ .../Services/S3WriteStream_20230907130949.cs | 108 ++++++ .../Services/S3WriteStream_20230907131020.cs | 108 ++++++ .../Services/S3WriteStream_20230907131032.cs | 108 ++++++ .../Services/S3WriteStream_20230907131048.cs | 109 ++++++ .../Services/S3WriteStream_20230907131526.cs | 110 ++++++ .../Services/S3WriteStream_20230907131529.cs | 113 ++++++ .../Services/S3WriteStream_20230907131704.cs | 116 ++++++ .../Services/S3WriteStream_20230907131835.cs | 114 ++++++ .../Services/S3WriteStream_20230907131840.cs | 113 ++++++ .../Services/S3WriteStream_20230907131855.cs | 114 ++++++ .../Services/S3WriteStream_20230907131914.cs | 115 ++++++ .../Services/S3WriteStream_20230907131918.cs | 115 ++++++ .../Services/S3WriteStream_20230907132115.cs | 120 ++++++ .../Services/S3WriteStream_20230907132117.cs | 120 ++++++ .../Services/S3WriteStream_20230907132230.cs | 120 ++++++ .../Services/S3WriteStream_20230907132231.cs | 120 ++++++ .../Services/S3WriteStream_20230907132240.cs | 120 ++++++ .../Services/S3WriteStream_20230907132242.cs | 120 ++++++ .../Services/S3WriteStream_20230907132338.cs | 120 ++++++ .../Services/S3WriteStream_20230907132350.cs | 125 +++++++ .../Services/S3WriteStream_20230907132400.cs | 125 +++++++ .../Services/S3WriteStream_20230907132417.cs | 125 +++++++ .../Services/S3WriteStream_20230907132419.cs | 124 +++++++ .../Services/S3WriteStream_20230907132431.cs | 125 +++++++ .../Services/S3WriteStream_20230907132432.cs | 124 +++++++ .../Services/S3WriteStream_20230907132433.cs | 124 +++++++ .../Services/S3WriteStream_20230907132616.cs | 125 +++++++ .../Services/S3WriteStream_20230907132630.cs | 125 +++++++ .../Services/S3WriteStream_20230907132632.cs | 125 +++++++ .../Services/S3WriteStream_20230907132832.cs | 126 +++++++ .../Services/S3WriteStream_20230907132840.cs | 126 +++++++ .../Services/S3WriteStream_20230907133744.cs | 126 +++++++ .../Services/S3WriteStream_20230907140811.cs | 127 +++++++ .../Services/S3WriteStream_20230907140817.cs | 151 ++++++++ .../Services/S3WriteStream_20230907140826.cs | 151 ++++++++ .../Services/S3WriteStream_20230907141040.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141052.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141053.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141054.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141056.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141058.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141059.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141100.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141102.cs | 155 ++++++++ .../Services/S3WriteStream_20230907142255.cs | 153 ++++++++ .../Services/S3WriteStream_20230907171436.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171441.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171450.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171526.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171534.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171604.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171626.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171627.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171628.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171630.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171636.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171638.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171720.cs | 161 ++++++++ .../Services/S3WriteStream_20230907171734.cs | 161 ++++++++ .../Services/S3WriteStream_20230907171736.cs | 161 ++++++++ .../Services/S3WriteStream_20230907172014.cs | 162 +++++++++ .../Services/S3WriteStream_20230907172018.cs | 163 +++++++++ .../Services/S3WriteStream_20230907172019.cs | 163 +++++++++ .../Services/S3WriteStream_20230907172038.cs | 163 +++++++++ .../Services/S3WriteStream_20230907172039.cs | 163 +++++++++ .../Services/S3WriteStream_20230908125336.cs | 163 +++++++++ .../Services/S3WriteStream_20230908125337.cs | 163 +++++++++ .../Services/S3WriteStream_20230908125344.cs | 163 +++++++++ .../SharedFileService_20230906141540.cs | 92 +++++ .../SharedFileService_20230907104153.cs | 92 +++++ .../SharedFileService_20230907134349.cs | 92 +++++ .../SharedFileService_20230907134355.cs | 92 +++++ .../SharedFileService_20230907171909.cs | 95 +++++ .../SharedFileService_20230907171920.cs | 97 +++++ .../SharedFileService_20230907171930.cs | 98 +++++ .../SharedFileService_20230907171931.cs | 98 +++++ .../SharedFileService_20230907171955.cs | 98 +++++ .../SharedFileService_20230907171956.cs | 98 +++++ .../SharedFileService_20230908125406.cs | 96 +++++ .../SharedFileService_20230908125407.cs | 96 +++++ ...SmtTransferEngineService_20230907084301.cs | 181 +++++++++ ...SmtTransferEngineService_20230908161128.cs | 183 ++++++++++ ...SmtTransferEngineService_20230908161129.cs | 183 ++++++++++ ...SmtTransferEngineService_20230908161150.cs | 183 ++++++++++ ...SmtTransferEngineService_20230908161950.cs | 183 ++++++++++ ...SmtTransferEngineService_20230908162047.cs | 184 ++++++++++ ...SmtTransferEngineService_20230908162317.cs | 188 ++++++++++ ...SmtTransferEngineService_20230908162334.cs | 188 ++++++++++ ...SmtTransferEngineService_20230908162336.cs | 188 ++++++++++ ...SmtTransferEngineService_20230908162356.cs | 188 ++++++++++ ...SmtTransferEngineService_20230908162358.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911083245.cs | 190 ++++++++++ ...SmtTransferEngineService_20230911083247.cs | 190 ++++++++++ ...SmtTransferEngineService_20230911083620.cs | 189 ++++++++++ ...SmtTransferEngineService_20230911083625.cs | 189 ++++++++++ ...SmtTransferEngineService_20230911083627.cs | 189 ++++++++++ ...SmtTransferEngineService_20230911083628.cs | 189 ++++++++++ ...SmtTransferEngineService_20230911083939.cs | 190 ++++++++++ ...SmtTransferEngineService_20230911084126.cs | 192 ++++++++++ ...SmtTransferEngineService_20230911084129.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084135.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084136.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084149.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084150.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084151.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084152.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084153.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084154.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084155.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084156.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084157.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084158.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084159.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084200.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084201.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084202.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084203.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084204.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084205.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084231.cs | 189 ++++++++++ ...SmtTransferEngineService_20230911084236.cs | 191 ++++++++++ ...SmtTransferEngineService_20230911084237.cs | 190 ++++++++++ ...SmtTransferEngineService_20230911084410.cs | 190 ++++++++++ ...SmtTransferEngineService_20230911084539.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084556.cs | 185 ++++++++++ ...SmtTransferEngineService_20230911084557.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084610.cs | 185 ++++++++++ ...SmtTransferEngineService_20230911084658.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084700.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084701.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084711.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084713.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084838.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084843.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084844.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084845.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084846.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911085524.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085526.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085550.cs | 175 +++++++++ ...SmtTransferEngineService_20230911085551.cs | 174 +++++++++ ...SmtTransferEngineService_20230911085557.cs | 175 +++++++++ ...SmtTransferEngineService_20230911085558.cs | 174 +++++++++ ...SmtTransferEngineService_20230911085704.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085706.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085712.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085757.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085803.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085805.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085809.cs | 173 +++++++++ ...slationEngineServiceBase_20230907084301.cs | 212 +++++++++++ ...slationEngineServiceBase_20230911085214.cs | 219 +++++++++++ ...slationEngineServiceBase_20230911085221.cs | 219 +++++++++++ ...slationEngineServiceBase_20230911085339.cs | 227 ++++++++++++ ...slationEngineServiceBase_20230911085341.cs | 227 ++++++++++++ ...slationEngineServiceBase_20230911085421.cs | 227 ++++++++++++ ...slationEngineServiceBase_20230911085505.cs | 227 ++++++++++++ ...slationEngineServiceBase_20230911085623.cs | 230 ++++++++++++ ...slationEngineServiceBase_20230911085631.cs | 227 ++++++++++++ ...slationEngineServiceBase_20230911085641.cs | 228 ++++++++++++ ...slationEngineServiceBase_20230911085646.cs | 228 ++++++++++++ ...slationEngineServiceBase_20230911085651.cs | 228 ++++++++++++ ...slationEngineServiceBase_20230911085722.cs | 228 ++++++++++++ ...slationEngineServiceBase_20230911085823.cs | 228 ++++++++++++ .../Usings_20230907084316.cs | 47 +++ .../Usings_20230907131743.cs | 47 +++ .../Usings_20230907141855.cs | 49 +++ .../Usings_20230907141856.cs | 48 +++ .../Usings_20230907141900.cs | 48 +++ .../Usings_20230907141901.cs | 48 +++ ...rMLNmtEngineServiceTests_20230906141540.cs | 192 ++++++++++ ...rMLNmtEngineServiceTests_20230907185104.cs | 192 ++++++++++ 220 files changed, 35526 insertions(+) create mode 100644 .history/.gitignore_20230911115750 create mode 100644 .history/.gitignore_20230911115828 create mode 100644 .history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs create mode 100644 .history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs create mode 100644 .history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs diff --git a/.history/.gitignore_20230911115750 b/.history/.gitignore_20230911115750 new file mode 100644 index 000000000..e1f430ccb --- /dev/null +++ b/.history/.gitignore_20230911115750 @@ -0,0 +1,51 @@ + +#ignore thumbnails created by windows +Thumbs.db +#Ignore files build by Visual Studio +*.obj +*.exe +*.pdb +*.user +*.aps +*.pch +*.vspscc +*_i.c +*_p.c +*.ncb +*.suo +*.tlb +*.tlh +*.bak +*.cache +*.ilk +*.log +[Bb]in +[Dd]ebug*/ +*.lib +*.sbr +obj/ +[Rr]elease*/ +_ReSharper*/ +[Tt]est[Rr]esult* + +*.Resharper +*.DotSettings +*- Copy* +packages/* +!packages/repositories.config +lib/ +*.files +*.VC.db +*.VC.opendb +*.lock.json +.vs +appsettings.user.json +artifacts +!**/Tes/release/ +thot-new-model.zip +*.tgz +out/ +src/sentencepiece4c/build/ +src/sentencepiece4cbuild/ +!samples/**/release +**/*.feature.cs diff --git a/.history/.gitignore_20230911115828 b/.history/.gitignore_20230911115828 new file mode 100644 index 000000000..fd44de4b5 --- /dev/null +++ b/.history/.gitignore_20230911115828 @@ -0,0 +1,52 @@ + +#ignore thumbnails created by windows +Thumbs.db +#Ignore files build by Visual Studio +*.obj +*.exe +*.pdb +*.user +*.aps +*.pch +*.vspscc +*_i.c +*_p.c +*.ncb +*.suo +*.tlb +*.tlh +*.bak +*.cache +*.ilk +*.log +[Bb]in +[Dd]ebug*/ +*.lib +*.sbr +obj/ +[Rr]elease*/ +_ReSharper*/ +[Tt]est[Rr]esult* +.history* + +*.Resharper +*.DotSettings +*- Copy* +packages/* +!packages/repositories.config +lib/ +*.files +*.VC.db +*.VC.opendb +*.lock.json +.vs +appsettings.user.json +artifacts +!**/Tes/release/ +thot-new-model.zip +*.tgz +out/ +src/sentencepiece4c/build/ +src/sentencepiece4cbuild/ +!samples/**/release +**/*.feature.cs diff --git a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs b/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs new file mode 100644 index 000000000..a8c7f1fd4 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs @@ -0,0 +1,317 @@ +using Microsoft.AspNetCore.Http; +using Serval.Translation.V1; + +namespace Microsoft.Extensions.DependencyInjection; + +public static class IMachineBuilderExtensions +{ + public static IMachineBuilder AddServiceOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddServiceOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddSmtTransferEngineOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddSmtTransferEngineOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddClearMLNmtEngineOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddClearMLNmtEngineOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddSharedFileOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddSharedFileOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddThotSmtModel( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder.AddThotSmtModel(); + } + + public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder.AddThotSmtModel(); + } + + public static IMachineBuilder AddTransferEngine(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddUnigramTruecaser(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddClearMLService(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); + + // workaround register satisfying the interface and as a hosted service. + builder.Services.AddSingleton(); + builder.Services.AddHostedService(p => p.GetRequiredService()); + + builder.Services + .AddHttpClient() + .AddTransientHttpErrorPolicy( + b => b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) + ); + + return builder; + } + + public static IMachineBuilder AddMongoBackgroundJobClient( + this IMachineBuilder builder, + string? connectionString = null + ) + { + builder.Services.AddHangfire( + c => + c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) + .UseSimpleAssemblyNameTypeSerializer() + .UseRecommendedSerializerSettings() + .UseMongoStorage( + connectionString ?? builder.Configuration.GetConnectionString("Hangfire"), + new MongoStorageOptions + { + MigrationOptions = new MongoMigrationOptions + { + MigrationStrategy = new MigrateMongoMigrationStrategy(), + BackupStrategy = new CollectionMongoBackupStrategy() + }, + CheckConnection = true, + CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, + } + ) + ); + builder.Services.AddHealthChecks().AddCheck(name: "Hangfire"); + return builder; + } + + public static IMachineBuilder AddBackgroundJobServer( + this IMachineBuilder builder, + IEnumerable? engineTypes = null + ) + { + engineTypes ??= + builder.Configuration?.GetSection("TranslationEngines").Get() + ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; + var queues = new List(); + foreach (TranslationEngineType engineType in engineTypes.Distinct()) + { + switch (engineType) + { + case TranslationEngineType.SmtTransfer: + builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); + queues.Add("smt_transfer"); + break; + case TranslationEngineType.Nmt: + builder.AddClearMLService(); + queues.Add("nmt"); + break; + } + } + + builder.Services.AddHangfireServer(o => + { + o.Queues = queues.ToArray(); + }); + return builder; + } + + public static IMachineBuilder AddMemoryDataAccess(this IMachineBuilder builder) + { + builder.Services.AddMemoryDataAccess(o => + { + o.AddRepository(); + o.AddRepository(); + o.AddRepository(); + }); + + return builder; + } + + public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, string? connectionString = null) + { + connectionString ??= builder.Configuration.GetConnectionString("Mongo"); + builder.Services.AddMongoDataAccess( + connectionString, + "SIL.Machine.AspNetCore.Models", + o => + { + o.AddRepository( + "translation_engines", + init: c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(p => p.EngineId) + ) + ) + ); + o.AddRepository( + "locks", + init: async c => + { + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending("writerLock._id")) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending("readerLocks._id")) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending("writerQueue._id")) + ); + } + ); + o.AddRepository( + "train_segment_pairs", + init: c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) + ) + ) + ); + } + ); + builder.Services.AddHealthChecks().AddMongoDb(connectionString, name: "Mongo"); + + return builder; + } + + public static IMachineBuilder AddServalPlatformService( + this IMachineBuilder builder, + string? connectionString = null + ) + { + builder.Services.AddScoped(); + builder.Services + .AddGrpcClient(o => + { + o.Address = new Uri(connectionString ?? builder.Configuration.GetConnectionString("Serval")); + }) + .ConfigureChannel(o => + { + o.MaxRetryAttempts = null; + o.ServiceConfig = new ServiceConfig + { + MethodConfigs = + { + new MethodConfig + { + Names = { MethodName.Default }, + RetryPolicy = new RetryPolicy + { + MaxAttempts = 10, + InitialBackoff = TimeSpan.FromSeconds(1), + MaxBackoff = TimeSpan.FromSeconds(5), + BackoffMultiplier = 1.5, + RetryableStatusCodes = { StatusCode.Unavailable } + } + }, + new MethodConfig + { + Names = + { + new MethodName + { + Service = "serval.translation.v1.TranslationPlatformApi", + Method = "UpdateBuildStatus" + } + } + }, + } + }; + }); + + return builder; + } + + public static IMachineBuilder AddServalTranslationEngineService( + this IMachineBuilder builder, + string? connectionString = null, + IEnumerable? engineTypes = null + ) + { + builder.Services.AddGrpc(options => options.Interceptors.Add()); + builder.AddServalPlatformService(connectionString ?? builder.Configuration.GetConnectionString("Serval")); + engineTypes ??= + builder.Configuration?.GetSection("TranslationEngines").Get() + ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; + foreach (TranslationEngineType engineType in engineTypes.Distinct()) + { + switch (engineType) + { + case TranslationEngineType.SmtTransfer: + builder.Services.AddSingleton(); + builder.Services.AddHostedService(); + builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); + builder.Services.AddScoped(); + break; + case TranslationEngineType.Nmt: + builder.AddClearMLService(); + builder.Services.AddScoped(); + break; + } + } + builder.Services.AddGrpcHealthChecks(); + + return builder; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs b/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs new file mode 100644 index 000000000..739a72924 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs @@ -0,0 +1,317 @@ +using Microsoft.AspNetCore.Http; +using Serval.Translation.V1; + +namespace Microsoft.Extensions.DependencyInjection; + +public static class IMachineBuilderExtensions +{ + public static IMachineBuilder AddServiceOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddServiceOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddSmtTransferEngineOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddSmtTransferEngineOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddClearMLNmtEngineOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddClearMLNmtEngineOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddSharedFileOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddSharedFileOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddThotSmtModel( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder.AddThotSmtModel(); + } + + public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder.AddThotSmtModel(); + } + + public static IMachineBuilder AddTransferEngine(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddUnigramTruecaser(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddClearMLService(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); + + // workaround register satisfying the interface and as a hosted service. + builder.Services.AddSingleton(); + builder.Services.AddHostedService(p => p.GetRequiredService()); + + builder.Services + .AddHttpClient() + .AddTransientHttpErrorPolicy( + b => b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) + ); + + return builder; + } + + public static IMachineBuilder AddMongoBackgroundJobClient( + this IMachineBuilder builder, + string? connectionString = null + ) + { + builder.Services.AddHangfire( + c => + c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) + .UseSimpleAssemblyNameTypeSerializer() + .UseRecommendedSerializerSettings() + .UseMongoStorage( + connectionString ?? builder.Configuration.GetConnectionString("Hangfire"), + new MongoStorageOptions + { + MigrationOptions = new MongoMigrationOptions + { + MigrationStrategy = new MigrateMongoMigrationStrategy(), + BackupStrategy = new CollectionMongoBackupStrategy() + }, + CheckConnection = true, + CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, + } + ) + ); + builder.Services.AddHealthChecks().AddCheck(name: "Hangfire"); + return builder; + } + + public static IMachineBuilder AddBackgroundJobServer( + this IMachineBuilder builder, + IEnumerable? engineTypes = null + ) + { + engineTypes ??= + builder.Configuration?.GetSection("TranslationEngines").Get() + ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; + var queues = new List(); + foreach (TranslationEngineType engineType in engineTypes.Distinct()) + { + switch (engineType) + { + case TranslationEngineType.SmtTransfer: + builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); + queues.Add("smt_transfer"); + break; + case TranslationEngineType.Nmt: + builder.AddClearMLService(); + queues.Add("nmt"); + break; + } + } + + builder.Services.AddHangfireServer(o => + { + o.Queues = queues.ToArray(); + }); + return builder; + } + + public static IMachineBuilder AddMemoryDataAccess(this IMachineBuilder builder) + { + builder.Services.AddMemoryDataAccess(o => + { + o.AddRepository(); + o.AddRepository(); + o.AddRepository(); + }); + + return builder; + } + + public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, string? connectionString = null) + { + connectionString ??= builder.Configuration.GetConnectionString("Mongo"); + builder.Services.AddMongoDataAccess( + connectionString, + "SIL.Machine.AspNetCore.Models", + o => + { + o.AddRepository( + "translation_engines", + init: c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(p => p.EngineId) + ) + ) + ); + o.AddRepository( + "locks", + init: async c => + { + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending("writerLock._id")) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending("readerLocks._id")) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending("writerQueue._id")) + ); + } + ); + o.AddRepository( + "train_segment_pairs", + init: c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) + ) + ) + ); + } + ); + builder.Services.AddHealthChecks().AddMongoDb(connectionString, name: "Mongo"); + + return builder; + } + + public static IMachineBuilder AddServalPlatformService( + this IMachineBuilder builder, + string? connectionString = null + ) + { + builder.Services.AddScoped(); + builder.Services + .AddGrpcClient(o => + { + o.Address = new Uri(connectionString ?? builder.Configuration.GetConnectionString("Serval")); + }) + .ConfigureChannel(o => + { + o.MaxRetryAttempts = null; + o.ServiceConfig = new ServiceConfig + { + MethodConfigs = + { + new MethodConfig + { + Names = { MethodName.Default }, + RetryPolicy = new Grpc.Net.Client.Configuration.RetryPolicy + { + MaxAttempts = 10, + InitialBackoff = TimeSpan.FromSeconds(1), + MaxBackoff = TimeSpan.FromSeconds(5), + BackoffMultiplier = 1.5, + RetryableStatusCodes = { StatusCode.Unavailable } + } + }, + new MethodConfig + { + Names = + { + new MethodName + { + Service = "serval.translation.v1.TranslationPlatformApi", + Method = "UpdateBuildStatus" + } + } + }, + } + }; + }); + + return builder; + } + + public static IMachineBuilder AddServalTranslationEngineService( + this IMachineBuilder builder, + string? connectionString = null, + IEnumerable? engineTypes = null + ) + { + builder.Services.AddGrpc(options => options.Interceptors.Add()); + builder.AddServalPlatformService(connectionString ?? builder.Configuration.GetConnectionString("Serval")); + engineTypes ??= + builder.Configuration?.GetSection("TranslationEngines").Get() + ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; + foreach (TranslationEngineType engineType in engineTypes.Distinct()) + { + switch (engineType) + { + case TranslationEngineType.SmtTransfer: + builder.Services.AddSingleton(); + builder.Services.AddHostedService(); + builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); + builder.Services.AddScoped(); + break; + case TranslationEngineType.Nmt: + builder.AddClearMLService(); + builder.Services.AddScoped(); + break; + } + } + builder.Services.AddGrpcHealthChecks(); + + return builder; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs new file mode 100644 index 000000000..918f40b90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs new file mode 100644 index 000000000..918f40b90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs new file mode 100644 index 000000000..9e7eed761 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs @@ -0,0 +1,343 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + await sourceTrainWriter.BaseStream.DisposeAsync(); + await sourceTrainWriter.BaseStream.DisposeAsync(); + + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs new file mode 100644 index 000000000..e41df89d3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs @@ -0,0 +1,342 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + await sourceTrainWriter.BaseStream.DisposeAsync(); + await targetTrainWriter.BaseStream.DisposeAsync(); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs new file mode 100644 index 000000000..e41df89d3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs @@ -0,0 +1,342 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + await sourceTrainWriter.BaseStream.DisposeAsync(); + await targetTrainWriter.BaseStream.DisposeAsync(); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs new file mode 100644 index 000000000..e41df89d3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs @@ -0,0 +1,342 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + await sourceTrainWriter.BaseStream.DisposeAsync(); + await targetTrainWriter.BaseStream.DisposeAsync(); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs new file mode 100644 index 000000000..e41df89d3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs @@ -0,0 +1,342 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + await sourceTrainWriter.BaseStream.DisposeAsync(); + await targetTrainWriter.BaseStream.DisposeAsync(); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs new file mode 100644 index 000000000..bf3981d14 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs @@ -0,0 +1,342 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + await sourceTrainWriter.BaseStream.DisposeAsync(); + await targetTrainWriter.BaseStream.DisposeAsync(); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs new file mode 100644 index 000000000..a26cda901 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs new file mode 100644 index 000000000..918f40b90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs new file mode 100644 index 000000000..918f40b90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs new file mode 100644 index 000000000..918f40b90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs new file mode 100644 index 000000000..918f40b90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs new file mode 100644 index 000000000..918f40b90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs new file mode 100644 index 000000000..42eb064e1 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs @@ -0,0 +1,51 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineService : TranslationEngineServiceBase +{ + private readonly IClearMLService _clearMLService; + + public ClearMLNmtEngineService( + IBackgroundJobClient jobClient, + IPlatformService platformService, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + IRepository engines, + IClearMLService clearMLService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _clearMLService = clearMLService; + } + + public override TranslationEngineType Type => TranslationEngineType.Nmt; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + await _clearMLService.CreateProjectAsync(engineId, engineName, cancellationToken: CancellationToken.None); + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + string? projectId = await _clearMLService.GetProjectIdAsync(engineId, CancellationToken.None); + if (projectId is not null) + await _clearMLService.DeleteProjectAsync(projectId, CancellationToken.None); + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs new file mode 100644 index 000000000..42eb064e1 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs @@ -0,0 +1,51 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineService : TranslationEngineServiceBase +{ + private readonly IClearMLService _clearMLService; + + public ClearMLNmtEngineService( + IBackgroundJobClient jobClient, + IPlatformService platformService, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + IRepository engines, + IClearMLService clearMLService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _clearMLService = clearMLService; + } + + public override TranslationEngineType Type => TranslationEngineType.Nmt; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + await _clearMLService.CreateProjectAsync(engineId, engineName, cancellationToken: CancellationToken.None); + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + string? projectId = await _clearMLService.GetProjectIdAsync(engineId, CancellationToken.None); + if (projectId is not null) + await _clearMLService.DeleteProjectAsync(projectId, CancellationToken.None); + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs new file mode 100644 index 000000000..42eb064e1 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs @@ -0,0 +1,51 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineService : TranslationEngineServiceBase +{ + private readonly IClearMLService _clearMLService; + + public ClearMLNmtEngineService( + IBackgroundJobClient jobClient, + IPlatformService platformService, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + IRepository engines, + IClearMLService clearMLService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _clearMLService = clearMLService; + } + + public override TranslationEngineType Type => TranslationEngineType.Nmt; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + await _clearMLService.CreateProjectAsync(engineId, engineName, cancellationToken: CancellationToken.None); + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + string? projectId = await _clearMLService.GetProjectIdAsync(engineId, CancellationToken.None); + if (projectId is not null) + await _clearMLService.DeleteProjectAsync(projectId, CancellationToken.None); + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs new file mode 100644 index 000000000..56bbc9829 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs @@ -0,0 +1,95 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + return Task.FromResult( + new BufferedStream(new S3WriteStream(_client, objectId, _bucketName), 1024 * 1024 * 100) + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs new file mode 100644 index 000000000..56bbc9829 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs @@ -0,0 +1,95 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + return Task.FromResult( + new BufferedStream(new S3WriteStream(_client, objectId, _bucketName), 1024 * 1024 * 100) + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs new file mode 100644 index 000000000..aaf4a3f52 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs @@ -0,0 +1,97 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + 1024 * 1024 * 5 + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs new file mode 100644 index 000000000..66204bcaa --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs @@ -0,0 +1,97 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + 1024 * 1024 * 5)); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs new file mode 100644 index 000000000..413860de7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs @@ -0,0 +1,98 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + 1024 * 1024 * 5 + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs new file mode 100644 index 000000000..413860de7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs @@ -0,0 +1,98 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + 1024 * 1024 * 5 + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs new file mode 100644 index 000000000..1eeacbc76 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs @@ -0,0 +1,98 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + S3WriteStream.FiveMB + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs new file mode 100644 index 000000000..1eeacbc76 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs @@ -0,0 +1,98 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + S3WriteStream.FiveMB + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs new file mode 100644 index 000000000..2ae672c3c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs @@ -0,0 +1,107 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + private readonly ILoggerFactory _loggerFactory; + + public S3FileStorage( + string bucketName, + string basePath, + string accessKeyId, + string secretAccessKey, + string region, + ILoggerFactory loggerFactory + ) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + _loggerFactory = loggerFactory; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + S3WriteStream.FiveMB + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs new file mode 100644 index 000000000..0619ee785 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs @@ -0,0 +1,107 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + private readonly ILoggerFactory _loggerFactory; + + public S3FileStorage( + string bucketName, + string basePath, + string accessKeyId, + string secretAccessKey, + string region, + ILoggerFactory loggerFactory + ) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + _loggerFactory = loggerFactory; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + S3WriteStream.FiveMB + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs new file mode 100644 index 000000000..b5eb883cb --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs @@ -0,0 +1,108 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + private readonly ILoggerFactory _loggerFactory; + + public S3FileStorage( + string bucketName, + string basePath, + string accessKeyId, + string secretAccessKey, + string region, + ILoggerFactory loggerFactory + ) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + _loggerFactory = loggerFactory; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + S3WriteStream.FiveMB, + _loggerFactory + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs new file mode 100644 index 000000000..9058da211 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs @@ -0,0 +1,107 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + private readonly ILoggerFactory _loggerFactory; + + public S3FileStorage( + string bucketName, + string basePath, + string accessKeyId, + string secretAccessKey, + string region, + ILoggerFactory loggerFactory + ) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + _loggerFactory = loggerFactory; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId, _loggerFactory), + S3WriteStream.FiveMB + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs new file mode 100644 index 000000000..9058da211 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs @@ -0,0 +1,107 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + private readonly ILoggerFactory _loggerFactory; + + public S3FileStorage( + string bucketName, + string basePath, + string accessKeyId, + string secretAccessKey, + string region, + ILoggerFactory loggerFactory + ) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + _loggerFactory = loggerFactory; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId, _loggerFactory), + S3WriteStream.FiveMB + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs new file mode 100644 index 000000000..ff8563a42 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs @@ -0,0 +1,107 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + private readonly ILoggerFactory _loggerFactory; + + public S3FileStorage( + string bucketName, + string basePath, + string accessKeyId, + string secretAccessKey, + string region, + ILoggerFactory loggerFactory + ) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + _loggerFactory = loggerFactory; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId, _loggerFactory), + S3WriteStream.MaxPartSize + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs new file mode 100644 index 000000000..b2d5808f0 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs @@ -0,0 +1,76 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _bucketName; + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName) + { + _client = client; + _key = key; + _bucketName = bucketName; + _length = 0; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + await transferUtility.UploadAsync(uploadRequest); + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs new file mode 100644 index 000000000..e27f3416a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs @@ -0,0 +1,76 @@ +namespace SIL.Machine.AspNetCore.Services;S3Wri + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _bucketName; + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName) + { + _client = client; + _key = key; + _bucketName = bucketName; + _length = 0; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + await transferUtility.UploadAsync(uploadRequest); + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs new file mode 100644 index 000000000..b2d5808f0 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs @@ -0,0 +1,76 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _bucketName; + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName) + { + _client = client; + _key = key; + _bucketName = bucketName; + _length = 0; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + await transferUtility.UploadAsync(uploadRequest); + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs new file mode 100644 index 000000000..a4284a6fe --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs @@ -0,0 +1,77 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _bucketName; + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName) + { + _client = client; + _key = key; + _bucketName = bucketName; + _length = 0; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + await transferUtility.UploadAsync(uploadRequest); + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs new file mode 100644 index 000000000..effbfca90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs @@ -0,0 +1,79 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + await transferUtility.UploadAsync(uploadRequest); + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs new file mode 100644 index 000000000..1fcc576e6 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs @@ -0,0 +1,78 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + await transferUtility.UploadAsync(uploadRequest); + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs new file mode 100644 index 000000000..213ddd80a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs @@ -0,0 +1,78 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + await transferUtility.UploadAsync(uploadRequest); + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs new file mode 100644 index 000000000..24f757833 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs @@ -0,0 +1,87 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + FilePosition = _length + }; + _length += count; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs new file mode 100644 index 000000000..04454309b --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs @@ -0,0 +1,75 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + FilePosition = _length + }; + _length += count; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs new file mode 100644 index 000000000..46a493f7f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs @@ -0,0 +1,108 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + FilePosition = _length + }; + _length += count; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs new file mode 100644 index 000000000..46a493f7f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs @@ -0,0 +1,108 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + FilePosition = _length + }; + _length += count; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs new file mode 100644 index 000000000..a5cdde5c5 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs @@ -0,0 +1,106 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber + }; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs new file mode 100644 index 000000000..16f137a5a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs @@ -0,0 +1,107 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber + }; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs new file mode 100644 index 000000000..7d72d7d4d --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs @@ -0,0 +1,108 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + Stream = ms + }; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs new file mode 100644 index 000000000..21f06731c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs @@ -0,0 +1,108 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs new file mode 100644 index 000000000..21f06731c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs @@ -0,0 +1,108 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs new file mode 100644 index 000000000..c551af600 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs @@ -0,0 +1,109 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + UploadPartResponse response = await _client.UploadPartAsync(request); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs new file mode 100644 index 000000000..859618376 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs @@ -0,0 +1,110 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + UploadPartResponse response = await _client.UploadPartAsync(request); + if(response.HttpStatusCode != HttpStatusCode.OK) throw new HttpRequestException($"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}") + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs new file mode 100644 index 000000000..38f8cf6d7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs @@ -0,0 +1,113 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs new file mode 100644 index 000000000..4e5370496 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs @@ -0,0 +1,116 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + uploadRequest.StreamTransferProgress += new EventHandler( + UploadPartProgressEventCallback + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs new file mode 100644 index 000000000..e89e96826 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs @@ -0,0 +1,114 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler((_, e) => { }); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs new file mode 100644 index 000000000..7be03bf75 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs @@ -0,0 +1,113 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler((_, e) => { }); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs new file mode 100644 index 000000000..785cf5858 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs @@ -0,0 +1,114 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler((_, e) => { }); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs new file mode 100644 index 000000000..64918822a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs @@ -0,0 +1,115 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = (new LoggerFactory()).CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler((_, e) => { }); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs new file mode 100644 index 000000000..3a05d9ab0 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs @@ -0,0 +1,115 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler((_, e) => { }); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs new file mode 100644 index 000000000..2d1c7f47a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs @@ -0,0 +1,120 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs new file mode 100644 index 000000000..2d1c7f47a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs @@ -0,0 +1,120 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs new file mode 100644 index 000000000..7c5630f4b --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs @@ -0,0 +1,120 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs new file mode 100644 index 000000000..7c5630f4b --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs @@ -0,0 +1,120 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs new file mode 100644 index 000000000..52ecc2ef3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs @@ -0,0 +1,120 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs new file mode 100644 index 000000000..52ecc2ef3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs @@ -0,0 +1,120 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs new file mode 100644 index 000000000..cb6f6edb5 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs @@ -0,0 +1,120 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs new file mode 100644 index 000000000..d6bc7c0aa --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs @@ -0,0 +1,125 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs new file mode 100644 index 000000000..81005c673 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs @@ -0,0 +1,125 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs new file mode 100644 index 000000000..bff5aa1d7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs @@ -0,0 +1,125 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs new file mode 100644 index 000000000..ac7aa225c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs @@ -0,0 +1,124 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs new file mode 100644 index 000000000..5f5b59245 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs @@ -0,0 +1,125 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs new file mode 100644 index 000000000..530e928ca --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs @@ -0,0 +1,124 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs new file mode 100644 index 000000000..530e928ca --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs @@ -0,0 +1,124 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs new file mode 100644 index 000000000..7e13f6a62 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs @@ -0,0 +1,125 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs new file mode 100644 index 000000000..104dc7116 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs @@ -0,0 +1,125 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs new file mode 100644 index 000000000..104dc7116 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs @@ -0,0 +1,125 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs new file mode 100644 index 000000000..a97937131 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs @@ -0,0 +1,126 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs new file mode 100644 index 000000000..a97937131 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs @@ -0,0 +1,126 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs new file mode 100644 index 000000000..a97937131 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs @@ -0,0 +1,126 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs new file mode 100644 index 000000000..1024e4aec --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs @@ -0,0 +1,127 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) { } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs new file mode 100644 index 000000000..4ef04e880 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs @@ -0,0 +1,151 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs new file mode 100644 index 000000000..e4c5eb276 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs @@ -0,0 +1,151 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client.CompleteMultipartUpload(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs new file mode 100644 index 000000000..9a03bb2e4 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs new file mode 100644 index 000000000..584a0e09c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs @@ -0,0 +1,153 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs new file mode 100644 index 000000000..9b4f9cf96 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public static final int fiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs new file mode 100644 index 000000000..d7b0b6982 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public final static int fiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs new file mode 100644 index 000000000..a47284fb5 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public static const int fiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs new file mode 100644 index 000000000..50a58fafe --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int fiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs new file mode 100644 index 000000000..c13ee9bc2 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FIVE_MB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs new file mode 100644 index 000000000..c13ee9bc2 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FIVE_MB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs new file mode 100644 index 000000000..20cee4645 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMb = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs new file mode 100644 index 000000000..262f9a6c3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMbB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs new file mode 100644 index 000000000..9ea77dfee --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs new file mode 100644 index 000000000..9ea77dfee --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs new file mode 100644 index 000000000..6d25881bb --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs new file mode 100644 index 000000000..6d25881bb --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs new file mode 100644 index 000000000..95dd7899d --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs @@ -0,0 +1,161 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs new file mode 100644 index 000000000..001301acb --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs @@ -0,0 +1,161 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs new file mode 100644 index 000000000..001301acb --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs @@ -0,0 +1,161 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs new file mode 100644 index 000000000..7fc4c6f7a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs @@ -0,0 +1,162 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs new file mode 100644 index 000000000..afd96e3f9 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs @@ -0,0 +1,163 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + throw; + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs new file mode 100644 index 000000000..afd96e3f9 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs @@ -0,0 +1,163 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + throw; + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs new file mode 100644 index 000000000..339ca3a26 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs @@ -0,0 +1,163 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await AbortAsync(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + AbortAsync(e).WaitAndUnwrapException(); + throw; + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await AbortAsync(e); + } + } + + private async Task AbortAsync(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs new file mode 100644 index 000000000..339ca3a26 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs @@ -0,0 +1,163 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await AbortAsync(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + AbortAsync(e).WaitAndUnwrapException(); + throw; + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await AbortAsync(e); + } + } + + private async Task AbortAsync(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs new file mode 100644 index 000000000..fc2173053 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs @@ -0,0 +1,163 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int MaxPartSize = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = MaxPartSize + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await AbortAsync(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + AbortAsync(e).WaitAndUnwrapException(); + throw; + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await AbortAsync(e); + } + } + + private async Task AbortAsync(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs new file mode 100644 index 000000000..fc2173053 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs @@ -0,0 +1,163 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int MaxPartSize = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = MaxPartSize + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await AbortAsync(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + AbortAsync(e).WaitAndUnwrapException(); + throw; + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await AbortAsync(e); + } + } + + private async Task AbortAsync(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs new file mode 100644 index 000000000..fc2173053 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs @@ -0,0 +1,163 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int MaxPartSize = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = MaxPartSize + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await AbortAsync(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + AbortAsync(e).WaitAndUnwrapException(); + throw; + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await AbortAsync(e); + } + } + + private async Task AbortAsync(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs new file mode 100644 index 000000000..526cf95fd --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs @@ -0,0 +1,92 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + + public SharedFileService(IOptions? options = null) + { + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs new file mode 100644 index 000000000..526cf95fd --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs @@ -0,0 +1,92 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + + public SharedFileService(IOptions? options = null) + { + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs new file mode 100644 index 000000000..a7a778353 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs @@ -0,0 +1,92 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + + public SharedFileService(IOptions? options = null) + { + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + }} + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs new file mode 100644 index 000000000..526cf95fd --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs @@ -0,0 +1,92 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + + public SharedFileService(IOptions? options = null) + { + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs new file mode 100644 index 000000000..c7eb4f847 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs @@ -0,0 +1,95 @@ +using SIL.Reporting; + +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) + { + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs new file mode 100644 index 000000000..53777aeaa --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs @@ -0,0 +1,97 @@ +using SIL.Reporting; + +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) + { + _loggerFactory = loggerFactory; + + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs new file mode 100644 index 000000000..c9f4aa6b1 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs @@ -0,0 +1,98 @@ +using SIL.Reporting; + +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) + { + _loggerFactory = loggerFactory; + + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region, + _loggerFactory + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs new file mode 100644 index 000000000..c9f4aa6b1 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs @@ -0,0 +1,98 @@ +using SIL.Reporting; + +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) + { + _loggerFactory = loggerFactory; + + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region, + _loggerFactory + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs new file mode 100644 index 000000000..5804584c1 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs @@ -0,0 +1,98 @@ +using SIL.Reporting; + +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) + { + _loggerFactory = loggerFactory; + + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region, + _loggerFactory + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs new file mode 100644 index 000000000..5804584c1 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs @@ -0,0 +1,98 @@ +using SIL.Reporting; + +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) + { + _loggerFactory = loggerFactory; + + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region, + _loggerFactory + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs new file mode 100644 index 000000000..cef73bbeb --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs @@ -0,0 +1,96 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) + { + _loggerFactory = loggerFactory; + + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region, + _loggerFactory + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs new file mode 100644 index 000000000..cef73bbeb --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs @@ -0,0 +1,96 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) + { + _loggerFactory = loggerFactory; + + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region, + _loggerFactory + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs new file mode 100644 index 000000000..e7619bc63 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs @@ -0,0 +1,181 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs new file mode 100644 index 000000000..804a21c2e --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs @@ -0,0 +1,183 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + if (state.CurrentBuildRevision == -1) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs new file mode 100644 index 000000000..804a21c2e --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs @@ -0,0 +1,183 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + if (state.CurrentBuildRevision == -1) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs new file mode 100644 index 000000000..804a21c2e --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs @@ -0,0 +1,183 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + if (state.CurrentBuildRevision == -1) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs new file mode 100644 index 000000000..68d6dc96e --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs @@ -0,0 +1,183 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + if (state.CurrentBuildRevision == -1) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs new file mode 100644 index 000000000..38b27e746 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if (await Engines.GetAsync(e => e.Id == engineId && e.BuildId)) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs new file mode 100644 index 000000000..4c6134aa9 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if ( + ( + await Engines.GetAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs new file mode 100644 index 000000000..cf92eef1a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if ( + ( + await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs new file mode 100644 index 000000000..cf92eef1a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if ( + ( + await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs new file mode 100644 index 000000000..4672cf290 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if ( + !( + await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs new file mode 100644 index 000000000..4672cf290 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if ( + !( + await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs new file mode 100644 index 000000000..f75ec89f0 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs @@ -0,0 +1,190 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if ( + !( + await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs new file mode 100644 index 000000000..f75ec89f0 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs @@ -0,0 +1,190 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if ( + !( + await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs new file mode 100644 index 000000000..22727b2d7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs @@ -0,0 +1,189 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + if ( + !( + await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs new file mode 100644 index 000000000..22727b2d7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs @@ -0,0 +1,189 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + if ( + !( + await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs new file mode 100644 index 000000000..22727b2d7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs @@ -0,0 +1,189 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + if ( + !( + await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs new file mode 100644 index 000000000..22727b2d7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs @@ -0,0 +1,189 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + if ( + !( + await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs new file mode 100644 index 000000000..f5d00996f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs @@ -0,0 +1,190 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + if ( + !( + + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs new file mode 100644 index 000000000..dd3aa1037 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs @@ -0,0 +1,192 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach(var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if ( + !( + + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs new file mode 100644 index 000000000..b773a87c2 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!(engines).Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs new file mode 100644 index 000000000..46ad52609 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs @@ -0,0 +1,189 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + Console.WriteLine("|||||||||||||||||||"); + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs new file mode 100644 index 000000000..9991f0c87 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs @@ -0,0 +1,191 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + Console.WriteLine("|||||||||||||||||||"); + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + Console.WriteLine("|||||||||||||||||||"); + + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs new file mode 100644 index 000000000..3574fa79f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs @@ -0,0 +1,190 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + Console.WriteLine("|||||||||||||||||||"); + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + Console.WriteLine("|||||||||||||||||||"); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs new file mode 100644 index 000000000..47ce83b1f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs @@ -0,0 +1,190 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + Console.WriteLine("|||||||||||||||||||"); + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + Console.WriteLine("|||||||||||||||||||"); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs new file mode 100644 index 000000000..234a20433 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + Console.WriteLine("|||||||||||||||||||"); + IEnumerable engines = await Engines.GetAllAsync(e => e.Id == engineId); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + Console.WriteLine("|||||||||||||||||||"); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs new file mode 100644 index 000000000..92d3d3d1e --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs @@ -0,0 +1,185 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs new file mode 100644 index 000000000..26186680c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs new file mode 100644 index 000000000..eaa649a10 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs @@ -0,0 +1,185 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs new file mode 100644 index 000000000..76069459f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildRevision == 0 || engine.BuildState != BuildState.None || engine.Revision == 0) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs new file mode 100644 index 000000000..76069459f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildRevision == 0 || engine.BuildState != BuildState.None || engine.Revision == 0) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs new file mode 100644 index 000000000..76069459f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildRevision == 0 || engine.BuildState != BuildState.None || engine.Revision == 0) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs new file mode 100644 index 000000000..b73ba6ead --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.Revision == 0) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs new file mode 100644 index 000000000..b73ba6ead --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.Revision == 0) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs new file mode 100644 index 000000000..ebb403ba9 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.Revision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs new file mode 100644 index 000000000..296b73779 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs new file mode 100644 index 000000000..296b73779 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs new file mode 100644 index 000000000..296b73779 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs new file mode 100644 index 000000000..296b73779 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs new file mode 100644 index 000000000..3ab3f9d3f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs new file mode 100644 index 000000000..e4be91c25 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs new file mode 100644 index 000000000..91a738627 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs @@ -0,0 +1,175 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs new file mode 100644 index 000000000..e4eba1d38 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs @@ -0,0 +1,174 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs new file mode 100644 index 000000000..b4dde802b --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs @@ -0,0 +1,175 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); + + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs new file mode 100644 index 000000000..0eae7ff8b --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs @@ -0,0 +1,174 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs new file mode 100644 index 000000000..6bc79f5bd --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs new file mode 100644 index 000000000..6bc79f5bd --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs new file mode 100644 index 000000000..8538f7d87 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await G(engineetBuiltEngineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs new file mode 100644 index 000000000..6bc79f5bd --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs new file mode 100644 index 000000000..67a1023da --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngine(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs new file mode 100644 index 000000000..2cfccdc7e --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs new file mode 100644 index 000000000..2cfccdc7e --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs new file mode 100644 index 000000000..0f3d710af --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs @@ -0,0 +1,212 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs new file mode 100644 index 000000000..41ff71d4b --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs @@ -0,0 +1,219 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) + + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs new file mode 100644 index 000000000..d7acd2bfd --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs @@ -0,0 +1,219 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs new file mode 100644 index 000000000..2d7e5e928 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs @@ -0,0 +1,227 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs new file mode 100644 index 000000000..2d7e5e928 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs @@ -0,0 +1,227 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs new file mode 100644 index 000000000..2d7e5e928 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs @@ -0,0 +1,227 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs new file mode 100644 index 000000000..2d7e5e928 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs @@ -0,0 +1,227 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs new file mode 100644 index 000000000..8c2e0cbc3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs @@ -0,0 +1,230 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task ThrowExceptionIfEngineIsNotBuilt( + string engineId, + CancellationToken cancellationToken + ) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs new file mode 100644 index 000000000..f310e85f3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs @@ -0,0 +1,227 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs new file mode 100644 index 000000000..dc702f8de --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs @@ -0,0 +1,228 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs new file mode 100644 index 000000000..dc702f8de --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs @@ -0,0 +1,228 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs new file mode 100644 index 000000000..d6511bf11 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs @@ -0,0 +1,228 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs new file mode 100644 index 000000000..dc702f8de --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs @@ -0,0 +1,228 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs new file mode 100644 index 000000000..dc702f8de --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs @@ -0,0 +1,228 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs new file mode 100644 index 000000000..21868db71 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs @@ -0,0 +1,47 @@ +global using System.Collections; +global using System.Collections.Concurrent; +global using System.Diagnostics; +global using System.Diagnostics.CodeAnalysis; +global using System.IO.Compression; +global using System.Linq.Expressions; +global using System.Net; +global using System.Reflection; +global using System.Runtime.CompilerServices; +global using System.Text; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Amazon; +global using Amazon.S3; +global using Amazon.S3.Model; +global using Amazon.S3.Transfer; +global using Grpc.Core; +global using Grpc.Core.Interceptors; +global using Grpc.Net.Client.Configuration; +global using Hangfire; +global using Hangfire.Mongo; +global using Hangfire.Mongo.Migration.Strategies; +global using Hangfire.Mongo.Migration.Strategies.Backup; +global using Microsoft.AspNetCore.Routing; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Diagnostics.HealthChecks; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using MongoDB.Driver; +global using MongoDB.Driver.Linq; +global using Nito.AsyncEx; +global using Polly; +global using SIL.DataAccess; +global using SIL.Machine.AspNetCore.Configuration; +global using SIL.Machine.AspNetCore.Models; +global using SIL.Machine.AspNetCore.Services; +global using SIL.Machine.AspNetCore.Utils; +global using SIL.Machine.Corpora; +global using SIL.Machine.Morphology.HermitCrab; +global using SIL.Machine.Tokenization; +global using SIL.Machine.Translation; +global using SIL.Machine.Translation.Thot; +global using SIL.Machine.Utils; +global using SIL.ObjectModel; +global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs new file mode 100644 index 000000000..0b07fae03 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs @@ -0,0 +1,47 @@ +global using System.Collections; +global using System.Collections.Concurrent; +global using System.Diagnostics; +global using System.Diagnostics.CodeAnalysis; +global using System.IO.Compression; +global using System.Linq.Expressions; +global using System.Net; +global using System.Reflection; +global using System.Runtime.CompilerServices; +global using System.Text; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Amazon; +global using Amazon.S3; +global using Amazon.S3.Model; +global using Amazon.Runtime; +global using Grpc.Core; +global using Grpc.Core.Interceptors; +global using Grpc.Net.Client.Configuration; +global using Hangfire; +global using Hangfire.Mongo; +global using Hangfire.Mongo.Migration.Strategies; +global using Hangfire.Mongo.Migration.Strategies.Backup; +global using Microsoft.AspNetCore.Routing; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Diagnostics.HealthChecks; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using MongoDB.Driver; +global using MongoDB.Driver.Linq; +global using Nito.AsyncEx; +global using Polly; +global using SIL.DataAccess; +global using SIL.Machine.AspNetCore.Configuration; +global using SIL.Machine.AspNetCore.Models; +global using SIL.Machine.AspNetCore.Services; +global using SIL.Machine.AspNetCore.Utils; +global using SIL.Machine.Corpora; +global using SIL.Machine.Morphology.HermitCrab; +global using SIL.Machine.Tokenization; +global using SIL.Machine.Translation; +global using SIL.Machine.Translation.Thot; +global using SIL.Machine.Utils; +global using SIL.ObjectModel; +global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs new file mode 100644 index 000000000..a808510e6 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs @@ -0,0 +1,49 @@ +global using System.Collections; +global using System.Collections.Concurrent; +global using System.Diagnostics; +global using System.Diagnostics.CodeAnalysis; +global using System.IO.Compression; +global using System.Linq.Expressions; +global using System.Net; +global using System.Reflection; +global using System.Runtime.CompilerServices; +global using System.Text; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Amazon; +global using Amazon.S3; +global using Amazon.S3.Model; +global using Amazon.Runtime; +global using Grpc.Core; +global using Grpc.Core.Interceptors; +global using Grpc.Net.Client.Configuration; +global using Hangfire; +global using Hangfire.Mongo; +global using Hangfire.Mongo.Migration.Strategies; +global using Hangfire.Mongo.Migration.Strategies.Backup; +global using Microsoft.AspNetCore.Routing; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Diagnostics.HealthChecks; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using MongoDB.Driver; +global using MongoDB.Driver.Linq; +global using Nito.AsyncEx; +using Nito.AsyncEx.Synchronous; + +global using Polly; +global using SIL.DataAccess; +global using SIL.Machine.AspNetCore.Configuration; +global using SIL.Machine.AspNetCore.Models; +global using SIL.Machine.AspNetCore.Services; +global using SIL.Machine.AspNetCore.Utils; +global using SIL.Machine.Corpora; +global using SIL.Machine.Morphology.HermitCrab; +global using SIL.Machine.Tokenization; +global using SIL.Machine.Translation; +global using SIL.Machine.Translation.Thot; +global using SIL.Machine.Utils; +global using SIL.ObjectModel; +global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs new file mode 100644 index 000000000..981c59ef5 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs @@ -0,0 +1,48 @@ +global using System.Collections; +global using System.Collections.Concurrent; +global using System.Diagnostics; +global using System.Diagnostics.CodeAnalysis; +global using System.IO.Compression; +global using System.Linq.Expressions; +global using System.Net; +global using System.Reflection; +global using System.Runtime.CompilerServices; +global using System.Text; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Amazon; +global using Amazon.S3; +global using Amazon.S3.Model; +global using Amazon.Runtime; +global using Grpc.Core; +global using Grpc.Core.Interceptors; +global using Grpc.Net.Client.Configuration; +global using Hangfire; +global using Hangfire.Mongo; +global using Hangfire.Mongo.Migration.Strategies; +global using Hangfire.Mongo.Migration.Strategies.Backup; +global using Microsoft.AspNetCore.Routing; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Diagnostics.HealthChecks; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using MongoDB.Driver; +global using MongoDB.Driver.Linq; +global using Nito.AsyncEx; +using Nito.AsyncEx.Synchronous; +global using Polly; +global using SIL.DataAccess; +global using SIL.Machine.AspNetCore.Configuration; +global using SIL.Machine.AspNetCore.Models; +global using SIL.Machine.AspNetCore.Services; +global using SIL.Machine.AspNetCore.Utils; +global using SIL.Machine.Corpora; +global using SIL.Machine.Morphology.HermitCrab; +global using SIL.Machine.Tokenization; +global using SIL.Machine.Translation; +global using SIL.Machine.Translation.Thot; +global using SIL.Machine.Utils; +global using SIL.ObjectModel; +global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs new file mode 100644 index 000000000..8544d1e96 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs @@ -0,0 +1,48 @@ +global using System.Collections; +global using System.Collections.Concurrent; +global using System.Diagnostics; +global using System.Diagnostics.CodeAnalysis; +global using System.IO.Compression; +global using System.Linq.Expressions; +global using System.Net; +global using System.Reflection; +global using System.Runtime.CompilerServices; +global using System.Text; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Amazon; +global using Amazon.S3; +global using Amazon.S3.Model; +global using Amazon.Runtime; +global using Grpc.Core; +global using Grpc.Core.Interceptors; +global using Grpc.Net.Client.Configuration; +global using Hangfire; +global using Hangfire.Mongo; +global using Hangfire.Mongo.Migration.Strategies; +global using Hangfire.Mongo.Migration.Strategies.Backup; +global using Microsoft.AspNetCore.Routing; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Diagnostics.HealthChecks; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using MongoDB.Driver; +global using MongoDB.Driver.Linq; +global using Nito.AsyncEx; +global using Nito.AsyncEx.Synchronous; +global using Polly; +global using SIL.DataAccess; +global using SIL.Machine.AspNetCore.Configuration; +global using SIL.Machine.AspNetCore.Models; +global using SIL.Machine.AspNetCore.Services; +global using SIL.Machine.AspNetCore.Utils; +global using SIL.Machine.Corpora; +global using SIL.Machine.Morphology.HermitCrab; +global using SIL.Machine.Tokenization; +global using SIL.Machine.Translation; +global using SIL.Machine.Translation.Thot; +global using SIL.Machine.Utils; +global using SIL.ObjectModel; +global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs new file mode 100644 index 000000000..8544d1e96 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs @@ -0,0 +1,48 @@ +global using System.Collections; +global using System.Collections.Concurrent; +global using System.Diagnostics; +global using System.Diagnostics.CodeAnalysis; +global using System.IO.Compression; +global using System.Linq.Expressions; +global using System.Net; +global using System.Reflection; +global using System.Runtime.CompilerServices; +global using System.Text; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Amazon; +global using Amazon.S3; +global using Amazon.S3.Model; +global using Amazon.Runtime; +global using Grpc.Core; +global using Grpc.Core.Interceptors; +global using Grpc.Net.Client.Configuration; +global using Hangfire; +global using Hangfire.Mongo; +global using Hangfire.Mongo.Migration.Strategies; +global using Hangfire.Mongo.Migration.Strategies.Backup; +global using Microsoft.AspNetCore.Routing; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Diagnostics.HealthChecks; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using MongoDB.Driver; +global using MongoDB.Driver.Linq; +global using Nito.AsyncEx; +global using Nito.AsyncEx.Synchronous; +global using Polly; +global using SIL.DataAccess; +global using SIL.Machine.AspNetCore.Configuration; +global using SIL.Machine.AspNetCore.Models; +global using SIL.Machine.AspNetCore.Services; +global using SIL.Machine.AspNetCore.Utils; +global using SIL.Machine.Corpora; +global using SIL.Machine.Morphology.HermitCrab; +global using SIL.Machine.Tokenization; +global using SIL.Machine.Translation; +global using SIL.Machine.Translation.Thot; +global using SIL.Machine.Utils; +global using SIL.ObjectModel; +global using SIL.WritingSystems; diff --git a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs b/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs new file mode 100644 index 000000000..f9d01a7d6 --- /dev/null +++ b/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs @@ -0,0 +1,192 @@ +namespace SIL.Machine.AspNetCore.Services; + +[TestFixture] +public class ClearMLNmtEngineServiceTests +{ + [Test] + public async Task CancelBuildAsync() + { + using var env = new TestEnvironment(); + env.ClearMLService + .CreateTaskAsync( + Arg.Any(), + "project1", + "engine1", + "es", + "en", + "memory:///", + Arg.Any() + ) + .Returns(Task.FromResult("task1")); + var task = new ClearMLTask + { + Id = "task1", + Project = new ClearMLProject { Id = "project1" }, + Status = ClearMLTaskStatus.InProgress + }; + bool first = true; + env.ClearMLService + .GetTaskByNameAsync(Arg.Any(), Arg.Any()) + .Returns(x => + { + if (first) + { + first = false; + return Task.FromResult(null); + } + return Task.FromResult(task); + }); + env.ClearMLService + .GetTaskByIdAsync("task1", Arg.Any()) + .Returns(Task.FromResult(task)); + await env.Service.StartBuildAsync("engine1", "build1", Array.Empty()); + await env.WaitForBuildToStartAsync(); + TranslationEngine engine = env.Engines.Get("engine1"); + Assert.That(engine.BuildState, Is.EqualTo(BuildState.Active)); + await env.Service.CancelBuildAsync("engine1"); + await env.WaitForBuildToFinishAsync(); + engine = env.Engines.Get("engine1"); + Assert.That(engine.BuildState, Is.EqualTo(BuildState.None)); + await env.ClearMLService.Received().StopTaskAsync("task1", Arg.Any()); + } + + private class TestEnvironment : DisposableBase + { + private readonly MemoryStorage _memoryStorage; + private readonly BackgroundJobClient _jobClient; + private BackgroundJobServer _jobServer; + private readonly IDistributedReaderWriterLockFactory _lockFactory; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + + public TestEnvironment() + { + Engines = new MemoryRepository(); + Engines.Add( + new TranslationEngine + { + Id = "engine1", + EngineId = "engine1", + SourceLanguage = "es", + TargetLanguage = "en" + } + ); + EngineOptions = new SmtTransferEngineOptions(); + _memoryStorage = new MemoryStorage(); + _jobClient = new BackgroundJobClient(_memoryStorage); + PlatformService = Substitute.For(); + ClearMLService = Substitute.For(); + ClearMLService + .GetProjectIdAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult("project1")); + _lockFactory = new DistributedReaderWriterLockFactory( + new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), + new MemoryRepository(), + new ObjectIdGenerator() + ); + _sharedFileService = new SharedFileService(); + _options = Substitute.For>(); + _options.CurrentValue.Returns( + new ClearMLNmtEngineOptions { BuildPollingTimeout = TimeSpan.FromMilliseconds(50) } + ); + _jobServer = CreateJobServer(); + Service = CreateService(); + } + + public ClearMLNmtEngineService Service { get; private set; } + public MemoryRepository Engines { get; } + public SmtTransferEngineOptions EngineOptions { get; } + public IPlatformService PlatformService { get; } + public IClearMLService ClearMLService { get; } + + public void StopServer() + { + _jobServer.Dispose(); + } + + public void StartServer() + { + _jobServer = CreateJobServer(); + Service = CreateService(); + } + + private BackgroundJobServer CreateJobServer() + { + var jobServerOptions = new BackgroundJobServerOptions + { + Activator = new EnvActivator(this), + Queues = new[] { "nmt" }, + CancellationCheckInterval = TimeSpan.FromMilliseconds(100), + }; + return new BackgroundJobServer(jobServerOptions, _memoryStorage); + } + + private ClearMLNmtEngineService CreateService() + { + return new ClearMLNmtEngineService( + _jobClient, + PlatformService, + _lockFactory, + new MemoryDataAccessContext(), + Engines, + ClearMLService + ); + } + + public Task WaitForBuildToFinishAsync() + { + return WaitForBuildState(e => e.BuildState is BuildState.None); + } + + public Task WaitForBuildToStartAsync() + { + return WaitForBuildState(e => e.BuildState is BuildState.Active); + } + + private async Task WaitForBuildState(Func predicate) + { + using ISubscription subscription = await Engines.SubscribeAsync( + e => e.EngineId == "engine1" + ); + while (true) + { + TranslationEngine? build = subscription.Change.Entity; + if (build is not null && predicate(build)) + break; + await subscription.WaitForChangeAsync(); + } + } + + protected override void DisposeManagedResources() + { + _jobServer.Dispose(); + } + + private class EnvActivator : JobActivator + { + private readonly TestEnvironment _env; + + public EnvActivator(TestEnvironment env) + { + _env = env; + } + + public override object ActivateJob(Type jobType) + { + if (jobType == typeof(ClearMLNmtEngineBuildJob)) + { + return new ClearMLNmtEngineBuildJob( + _env.PlatformService, + _env.Engines, + Substitute.For>(), + _env.ClearMLService, + _env._sharedFileService, + _env._options, + Substitute.For() + ); + } + return base.ActivateJob(jobType); + } + } + } +} diff --git a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs b/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs new file mode 100644 index 000000000..614dabc52 --- /dev/null +++ b/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs @@ -0,0 +1,192 @@ +namespace SIL.Machine.AspNetCore.Services; + +[TestFixture] +public class ClearMLNmtEngineServiceTests +{ + [Test] + public async Task CancelBuildAsync() + { + using var env = new TestEnvironment(); + env.ClearMLService + .CreateTaskAsync( + Arg.Any(), + "project1", + "engine1", + "es", + "en", + "memory:///", + Arg.Any() + ) + .Returns(Task.FromResult("task1")); + var task = new ClearMLTask + { + Id = "task1", + Project = new ClearMLProject { Id = "project1" }, + Status = ClearMLTaskStatus.InProgress + }; + bool first = true; + env.ClearMLService + .GetTaskByNameAsync(Arg.Any(), Arg.Any()) + .Returns(x => + { + if (first) + { + first = false; + return Task.FromResult(null); + } + return Task.FromResult(task); + }); + env.ClearMLService + .GetTaskByIdAsync("task1", Arg.Any()) + .Returns(Task.FromResult(task)); + await env.Service.StartBuildAsync("engine1", "build1", Array.Empty()); + await env.WaitForBuildToStartAsync(); + TranslationEngine engine = env.Engines.Get("engine1"); + Assert.That(engine.BuildState, Is.EqualTo(BuildState.Active)); + await env.Service.CancelBuildAsync("engine1"); + await env.WaitForBuildToFinishAsync(); + engine = env.Engines.Get("engine1"); + Assert.That(engine.BuildState, Is.EqualTo(BuildState.None)); + await env.ClearMLService.Received().StopTaskAsync("task1", Arg.Any()); + } + + private class TestEnvironment : DisposableBase + { + private readonly MemoryStorage _memoryStorage; + private readonly BackgroundJobClient _jobClient; + private BackgroundJobServer _jobServer; + private readonly IDistributedReaderWriterLockFactory _lockFactory; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + + public TestEnvironment() + { + Engines = new MemoryRepository(); + Engines.Add( + new TranslationEngine + { + Id = "engine1", + EngineId = "engine1", + SourceLanguage = "es", + TargetLanguage = "en" + } + ); + EngineOptions = new SmtTransferEngineOptions(); + _memoryStorage = new MemoryStorage(); + _jobClient = new BackgroundJobClient(_memoryStorage); + PlatformService = Substitute.For(); + ClearMLService = Substitute.For(); + ClearMLService + .GetProjectIdAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult("project1")); + _lockFactory = new DistributedReaderWriterLockFactory( + new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), + new MemoryRepository(), + new ObjectIdGenerator() + ); + _sharedFileService = new SharedFileService(Substitute.For()); + _options = Substitute.For>(); + _options.CurrentValue.Returns( + new ClearMLNmtEngineOptions { BuildPollingTimeout = TimeSpan.FromMilliseconds(50) } + ); + _jobServer = CreateJobServer(); + Service = CreateService(); + } + + public ClearMLNmtEngineService Service { get; private set; } + public MemoryRepository Engines { get; } + public SmtTransferEngineOptions EngineOptions { get; } + public IPlatformService PlatformService { get; } + public IClearMLService ClearMLService { get; } + + public void StopServer() + { + _jobServer.Dispose(); + } + + public void StartServer() + { + _jobServer = CreateJobServer(); + Service = CreateService(); + } + + private BackgroundJobServer CreateJobServer() + { + var jobServerOptions = new BackgroundJobServerOptions + { + Activator = new EnvActivator(this), + Queues = new[] { "nmt" }, + CancellationCheckInterval = TimeSpan.FromMilliseconds(100), + }; + return new BackgroundJobServer(jobServerOptions, _memoryStorage); + } + + private ClearMLNmtEngineService CreateService() + { + return new ClearMLNmtEngineService( + _jobClient, + PlatformService, + _lockFactory, + new MemoryDataAccessContext(), + Engines, + ClearMLService + ); + } + + public Task WaitForBuildToFinishAsync() + { + return WaitForBuildState(e => e.BuildState is BuildState.None); + } + + public Task WaitForBuildToStartAsync() + { + return WaitForBuildState(e => e.BuildState is BuildState.Active); + } + + private async Task WaitForBuildState(Func predicate) + { + using ISubscription subscription = await Engines.SubscribeAsync( + e => e.EngineId == "engine1" + ); + while (true) + { + TranslationEngine? build = subscription.Change.Entity; + if (build is not null && predicate(build)) + break; + await subscription.WaitForChangeAsync(); + } + } + + protected override void DisposeManagedResources() + { + _jobServer.Dispose(); + } + + private class EnvActivator : JobActivator + { + private readonly TestEnvironment _env; + + public EnvActivator(TestEnvironment env) + { + _env = env; + } + + public override object ActivateJob(Type jobType) + { + if (jobType == typeof(ClearMLNmtEngineBuildJob)) + { + return new ClearMLNmtEngineBuildJob( + _env.PlatformService, + _env.Engines, + Substitute.For>(), + _env.ClearMLService, + _env._sharedFileService, + _env._options, + Substitute.For() + ); + } + return base.ActivateJob(jobType); + } + } + } +} From 3fab687398ad4d5ab623675c844c45b043941ab0 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Mon, 11 Sep 2023 13:05:35 -0400 Subject: [PATCH 06/10] Revert "Empty commit to trigger workflow --ECL" This reverts commit c9384c86e4886efa63fa416e76571f0c88b7f0e6. --- .history/.gitignore_20230911115750 | 51 --- .history/.gitignore_20230911115828 | 52 --- ...MachineBuilderExtensions_20230907084316.cs | 317 ---------------- ...MachineBuilderExtensions_20230907133310.cs | 317 ---------------- ...ClearMLNmtEngineBuildJob_20230907084316.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907095831.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907135725.cs | 343 ------------------ ...ClearMLNmtEngineBuildJob_20230907135737.cs | 342 ----------------- ...ClearMLNmtEngineBuildJob_20230907135738.cs | 342 ----------------- ...ClearMLNmtEngineBuildJob_20230907135739.cs | 342 ----------------- ...ClearMLNmtEngineBuildJob_20230907140302.cs | 342 ----------------- ...ClearMLNmtEngineBuildJob_20230907140611.cs | 342 ----------------- ...ClearMLNmtEngineBuildJob_20230907140709.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907140719.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907140720.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907141110.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907141112.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907141905.cs | 340 ----------------- .../ClearMLNmtEngineService_20230906141540.cs | 51 --- .../ClearMLNmtEngineService_20230911085826.cs | 51 --- .../ClearMLNmtEngineService_20230911085829.cs | 51 --- .../Services/S3FileStorage_20230906141540.cs | 95 ----- .../Services/S3FileStorage_20230907104119.cs | 95 ----- .../Services/S3FileStorage_20230907130153.cs | 97 ----- .../Services/S3FileStorage_20230907130157.cs | 97 ----- .../Services/S3FileStorage_20230907132651.cs | 98 ----- .../Services/S3FileStorage_20230907132652.cs | 98 ----- .../Services/S3FileStorage_20230907171652.cs | 98 ----- .../Services/S3FileStorage_20230907171653.cs | 98 ----- .../Services/S3FileStorage_20230907171816.cs | 107 ------ .../Services/S3FileStorage_20230907171821.cs | 107 ------ .../Services/S3FileStorage_20230907171829.cs | 108 ------ .../Services/S3FileStorage_20230907171837.cs | 107 ------ .../Services/S3FileStorage_20230907171838.cs | 107 ------ .../Services/S3FileStorage_20230908125336.cs | 107 ------ .../Services/S3WriteStream_20230906141540.cs | 76 ---- .../Services/S3WriteStream_20230907104130.cs | 76 ---- .../Services/S3WriteStream_20230907104133.cs | 76 ---- .../Services/S3WriteStream_20230907130227.cs | 77 ---- .../Services/S3WriteStream_20230907130326.cs | 79 ---- .../Services/S3WriteStream_20230907130328.cs | 78 ---- .../Services/S3WriteStream_20230907130334.cs | 78 ---- .../Services/S3WriteStream_20230907130427.cs | 87 ----- .../Services/S3WriteStream_20230907130442.cs | 75 ---- .../Services/S3WriteStream_20230907130507.cs | 108 ------ .../Services/S3WriteStream_20230907130523.cs | 108 ------ .../Services/S3WriteStream_20230907130903.cs | 106 ------ .../Services/S3WriteStream_20230907130942.cs | 107 ------ .../Services/S3WriteStream_20230907130949.cs | 108 ------ .../Services/S3WriteStream_20230907131020.cs | 108 ------ .../Services/S3WriteStream_20230907131032.cs | 108 ------ .../Services/S3WriteStream_20230907131048.cs | 109 ------ .../Services/S3WriteStream_20230907131526.cs | 110 ------ .../Services/S3WriteStream_20230907131529.cs | 113 ------ .../Services/S3WriteStream_20230907131704.cs | 116 ------ .../Services/S3WriteStream_20230907131835.cs | 114 ------ .../Services/S3WriteStream_20230907131840.cs | 113 ------ .../Services/S3WriteStream_20230907131855.cs | 114 ------ .../Services/S3WriteStream_20230907131914.cs | 115 ------ .../Services/S3WriteStream_20230907131918.cs | 115 ------ .../Services/S3WriteStream_20230907132115.cs | 120 ------ .../Services/S3WriteStream_20230907132117.cs | 120 ------ .../Services/S3WriteStream_20230907132230.cs | 120 ------ .../Services/S3WriteStream_20230907132231.cs | 120 ------ .../Services/S3WriteStream_20230907132240.cs | 120 ------ .../Services/S3WriteStream_20230907132242.cs | 120 ------ .../Services/S3WriteStream_20230907132338.cs | 120 ------ .../Services/S3WriteStream_20230907132350.cs | 125 ------- .../Services/S3WriteStream_20230907132400.cs | 125 ------- .../Services/S3WriteStream_20230907132417.cs | 125 ------- .../Services/S3WriteStream_20230907132419.cs | 124 ------- .../Services/S3WriteStream_20230907132431.cs | 125 ------- .../Services/S3WriteStream_20230907132432.cs | 124 ------- .../Services/S3WriteStream_20230907132433.cs | 124 ------- .../Services/S3WriteStream_20230907132616.cs | 125 ------- .../Services/S3WriteStream_20230907132630.cs | 125 ------- .../Services/S3WriteStream_20230907132632.cs | 125 ------- .../Services/S3WriteStream_20230907132832.cs | 126 ------- .../Services/S3WriteStream_20230907132840.cs | 126 ------- .../Services/S3WriteStream_20230907133744.cs | 126 ------- .../Services/S3WriteStream_20230907140811.cs | 127 ------- .../Services/S3WriteStream_20230907140817.cs | 151 -------- .../Services/S3WriteStream_20230907140826.cs | 151 -------- .../Services/S3WriteStream_20230907141040.cs | 155 -------- .../Services/S3WriteStream_20230907141052.cs | 155 -------- .../Services/S3WriteStream_20230907141053.cs | 155 -------- .../Services/S3WriteStream_20230907141054.cs | 155 -------- .../Services/S3WriteStream_20230907141056.cs | 155 -------- .../Services/S3WriteStream_20230907141058.cs | 155 -------- .../Services/S3WriteStream_20230907141059.cs | 155 -------- .../Services/S3WriteStream_20230907141100.cs | 155 -------- .../Services/S3WriteStream_20230907141102.cs | 155 -------- .../Services/S3WriteStream_20230907142255.cs | 153 -------- .../Services/S3WriteStream_20230907171436.cs | 155 -------- .../Services/S3WriteStream_20230907171441.cs | 155 -------- .../Services/S3WriteStream_20230907171450.cs | 155 -------- .../Services/S3WriteStream_20230907171526.cs | 155 -------- .../Services/S3WriteStream_20230907171534.cs | 155 -------- .../Services/S3WriteStream_20230907171604.cs | 155 -------- .../Services/S3WriteStream_20230907171626.cs | 155 -------- .../Services/S3WriteStream_20230907171627.cs | 155 -------- .../Services/S3WriteStream_20230907171628.cs | 155 -------- .../Services/S3WriteStream_20230907171630.cs | 155 -------- .../Services/S3WriteStream_20230907171636.cs | 155 -------- .../Services/S3WriteStream_20230907171638.cs | 155 -------- .../Services/S3WriteStream_20230907171720.cs | 161 -------- .../Services/S3WriteStream_20230907171734.cs | 161 -------- .../Services/S3WriteStream_20230907171736.cs | 161 -------- .../Services/S3WriteStream_20230907172014.cs | 162 --------- .../Services/S3WriteStream_20230907172018.cs | 163 --------- .../Services/S3WriteStream_20230907172019.cs | 163 --------- .../Services/S3WriteStream_20230907172038.cs | 163 --------- .../Services/S3WriteStream_20230907172039.cs | 163 --------- .../Services/S3WriteStream_20230908125336.cs | 163 --------- .../Services/S3WriteStream_20230908125337.cs | 163 --------- .../Services/S3WriteStream_20230908125344.cs | 163 --------- .../SharedFileService_20230906141540.cs | 92 ----- .../SharedFileService_20230907104153.cs | 92 ----- .../SharedFileService_20230907134349.cs | 92 ----- .../SharedFileService_20230907134355.cs | 92 ----- .../SharedFileService_20230907171909.cs | 95 ----- .../SharedFileService_20230907171920.cs | 97 ----- .../SharedFileService_20230907171930.cs | 98 ----- .../SharedFileService_20230907171931.cs | 98 ----- .../SharedFileService_20230907171955.cs | 98 ----- .../SharedFileService_20230907171956.cs | 98 ----- .../SharedFileService_20230908125406.cs | 96 ----- .../SharedFileService_20230908125407.cs | 96 ----- ...SmtTransferEngineService_20230907084301.cs | 181 --------- ...SmtTransferEngineService_20230908161128.cs | 183 ---------- ...SmtTransferEngineService_20230908161129.cs | 183 ---------- ...SmtTransferEngineService_20230908161150.cs | 183 ---------- ...SmtTransferEngineService_20230908161950.cs | 183 ---------- ...SmtTransferEngineService_20230908162047.cs | 184 ---------- ...SmtTransferEngineService_20230908162317.cs | 188 ---------- ...SmtTransferEngineService_20230908162334.cs | 188 ---------- ...SmtTransferEngineService_20230908162336.cs | 188 ---------- ...SmtTransferEngineService_20230908162356.cs | 188 ---------- ...SmtTransferEngineService_20230908162358.cs | 188 ---------- ...SmtTransferEngineService_20230911083245.cs | 190 ---------- ...SmtTransferEngineService_20230911083247.cs | 190 ---------- ...SmtTransferEngineService_20230911083620.cs | 189 ---------- ...SmtTransferEngineService_20230911083625.cs | 189 ---------- ...SmtTransferEngineService_20230911083627.cs | 189 ---------- ...SmtTransferEngineService_20230911083628.cs | 189 ---------- ...SmtTransferEngineService_20230911083939.cs | 190 ---------- ...SmtTransferEngineService_20230911084126.cs | 192 ---------- ...SmtTransferEngineService_20230911084129.cs | 188 ---------- ...SmtTransferEngineService_20230911084135.cs | 188 ---------- ...SmtTransferEngineService_20230911084136.cs | 188 ---------- ...SmtTransferEngineService_20230911084149.cs | 188 ---------- ...SmtTransferEngineService_20230911084150.cs | 188 ---------- ...SmtTransferEngineService_20230911084151.cs | 188 ---------- ...SmtTransferEngineService_20230911084152.cs | 188 ---------- ...SmtTransferEngineService_20230911084153.cs | 188 ---------- ...SmtTransferEngineService_20230911084154.cs | 188 ---------- ...SmtTransferEngineService_20230911084155.cs | 188 ---------- ...SmtTransferEngineService_20230911084156.cs | 188 ---------- ...SmtTransferEngineService_20230911084157.cs | 188 ---------- ...SmtTransferEngineService_20230911084158.cs | 188 ---------- ...SmtTransferEngineService_20230911084159.cs | 188 ---------- ...SmtTransferEngineService_20230911084200.cs | 188 ---------- ...SmtTransferEngineService_20230911084201.cs | 188 ---------- ...SmtTransferEngineService_20230911084202.cs | 188 ---------- ...SmtTransferEngineService_20230911084203.cs | 188 ---------- ...SmtTransferEngineService_20230911084204.cs | 188 ---------- ...SmtTransferEngineService_20230911084205.cs | 188 ---------- ...SmtTransferEngineService_20230911084231.cs | 189 ---------- ...SmtTransferEngineService_20230911084236.cs | 191 ---------- ...SmtTransferEngineService_20230911084237.cs | 190 ---------- ...SmtTransferEngineService_20230911084410.cs | 190 ---------- ...SmtTransferEngineService_20230911084539.cs | 188 ---------- ...SmtTransferEngineService_20230911084556.cs | 185 ---------- ...SmtTransferEngineService_20230911084557.cs | 184 ---------- ...SmtTransferEngineService_20230911084610.cs | 185 ---------- ...SmtTransferEngineService_20230911084658.cs | 184 ---------- ...SmtTransferEngineService_20230911084700.cs | 184 ---------- ...SmtTransferEngineService_20230911084701.cs | 184 ---------- ...SmtTransferEngineService_20230911084711.cs | 184 ---------- ...SmtTransferEngineService_20230911084713.cs | 184 ---------- ...SmtTransferEngineService_20230911084838.cs | 184 ---------- ...SmtTransferEngineService_20230911084843.cs | 184 ---------- ...SmtTransferEngineService_20230911084844.cs | 184 ---------- ...SmtTransferEngineService_20230911084845.cs | 184 ---------- ...SmtTransferEngineService_20230911084846.cs | 184 ---------- ...SmtTransferEngineService_20230911085524.cs | 173 --------- ...SmtTransferEngineService_20230911085526.cs | 173 --------- ...SmtTransferEngineService_20230911085550.cs | 175 --------- ...SmtTransferEngineService_20230911085551.cs | 174 --------- ...SmtTransferEngineService_20230911085557.cs | 175 --------- ...SmtTransferEngineService_20230911085558.cs | 174 --------- ...SmtTransferEngineService_20230911085704.cs | 173 --------- ...SmtTransferEngineService_20230911085706.cs | 173 --------- ...SmtTransferEngineService_20230911085712.cs | 173 --------- ...SmtTransferEngineService_20230911085757.cs | 173 --------- ...SmtTransferEngineService_20230911085803.cs | 173 --------- ...SmtTransferEngineService_20230911085805.cs | 173 --------- ...SmtTransferEngineService_20230911085809.cs | 173 --------- ...slationEngineServiceBase_20230907084301.cs | 212 ----------- ...slationEngineServiceBase_20230911085214.cs | 219 ----------- ...slationEngineServiceBase_20230911085221.cs | 219 ----------- ...slationEngineServiceBase_20230911085339.cs | 227 ------------ ...slationEngineServiceBase_20230911085341.cs | 227 ------------ ...slationEngineServiceBase_20230911085421.cs | 227 ------------ ...slationEngineServiceBase_20230911085505.cs | 227 ------------ ...slationEngineServiceBase_20230911085623.cs | 230 ------------ ...slationEngineServiceBase_20230911085631.cs | 227 ------------ ...slationEngineServiceBase_20230911085641.cs | 228 ------------ ...slationEngineServiceBase_20230911085646.cs | 228 ------------ ...slationEngineServiceBase_20230911085651.cs | 228 ------------ ...slationEngineServiceBase_20230911085722.cs | 228 ------------ ...slationEngineServiceBase_20230911085823.cs | 228 ------------ .../Usings_20230907084316.cs | 47 --- .../Usings_20230907131743.cs | 47 --- .../Usings_20230907141855.cs | 49 --- .../Usings_20230907141856.cs | 48 --- .../Usings_20230907141900.cs | 48 --- .../Usings_20230907141901.cs | 48 --- ...rMLNmtEngineServiceTests_20230906141540.cs | 192 ---------- ...rMLNmtEngineServiceTests_20230907185104.cs | 192 ---------- 220 files changed, 35526 deletions(-) delete mode 100644 .history/.gitignore_20230911115750 delete mode 100644 .history/.gitignore_20230911115828 delete mode 100644 .history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs delete mode 100644 .history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs delete mode 100644 .history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs diff --git a/.history/.gitignore_20230911115750 b/.history/.gitignore_20230911115750 deleted file mode 100644 index e1f430ccb..000000000 --- a/.history/.gitignore_20230911115750 +++ /dev/null @@ -1,51 +0,0 @@ - -#ignore thumbnails created by windows -Thumbs.db -#Ignore files build by Visual Studio -*.obj -*.exe -*.pdb -*.user -*.aps -*.pch -*.vspscc -*_i.c -*_p.c -*.ncb -*.suo -*.tlb -*.tlh -*.bak -*.cache -*.ilk -*.log -[Bb]in -[Dd]ebug*/ -*.lib -*.sbr -obj/ -[Rr]elease*/ -_ReSharper*/ -[Tt]est[Rr]esult* - -*.Resharper -*.DotSettings -*- Copy* -packages/* -!packages/repositories.config -lib/ -*.files -*.VC.db -*.VC.opendb -*.lock.json -.vs -appsettings.user.json -artifacts -!**/Tes/release/ -thot-new-model.zip -*.tgz -out/ -src/sentencepiece4c/build/ -src/sentencepiece4cbuild/ -!samples/**/release -**/*.feature.cs diff --git a/.history/.gitignore_20230911115828 b/.history/.gitignore_20230911115828 deleted file mode 100644 index fd44de4b5..000000000 --- a/.history/.gitignore_20230911115828 +++ /dev/null @@ -1,52 +0,0 @@ - -#ignore thumbnails created by windows -Thumbs.db -#Ignore files build by Visual Studio -*.obj -*.exe -*.pdb -*.user -*.aps -*.pch -*.vspscc -*_i.c -*_p.c -*.ncb -*.suo -*.tlb -*.tlh -*.bak -*.cache -*.ilk -*.log -[Bb]in -[Dd]ebug*/ -*.lib -*.sbr -obj/ -[Rr]elease*/ -_ReSharper*/ -[Tt]est[Rr]esult* -.history* - -*.Resharper -*.DotSettings -*- Copy* -packages/* -!packages/repositories.config -lib/ -*.files -*.VC.db -*.VC.opendb -*.lock.json -.vs -appsettings.user.json -artifacts -!**/Tes/release/ -thot-new-model.zip -*.tgz -out/ -src/sentencepiece4c/build/ -src/sentencepiece4cbuild/ -!samples/**/release -**/*.feature.cs diff --git a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs b/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs deleted file mode 100644 index a8c7f1fd4..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs +++ /dev/null @@ -1,317 +0,0 @@ -using Microsoft.AspNetCore.Http; -using Serval.Translation.V1; - -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMachineBuilderExtensions -{ - public static IMachineBuilder AddServiceOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddServiceOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddClearMLNmtEngineOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddClearMLNmtEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddThotSmtModel( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder.AddThotSmtModel(); - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder.AddThotSmtModel(); - } - - public static IMachineBuilder AddTransferEngine(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddUnigramTruecaser(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddClearMLService(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); - - // workaround register satisfying the interface and as a hosted service. - builder.Services.AddSingleton(); - builder.Services.AddHostedService(p => p.GetRequiredService()); - - builder.Services - .AddHttpClient() - .AddTransientHttpErrorPolicy( - b => b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) - ); - - return builder; - } - - public static IMachineBuilder AddMongoBackgroundJobClient( - this IMachineBuilder builder, - string? connectionString = null - ) - { - builder.Services.AddHangfire( - c => - c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) - .UseSimpleAssemblyNameTypeSerializer() - .UseRecommendedSerializerSettings() - .UseMongoStorage( - connectionString ?? builder.Configuration.GetConnectionString("Hangfire"), - new MongoStorageOptions - { - MigrationOptions = new MongoMigrationOptions - { - MigrationStrategy = new MigrateMongoMigrationStrategy(), - BackupStrategy = new CollectionMongoBackupStrategy() - }, - CheckConnection = true, - CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, - } - ) - ); - builder.Services.AddHealthChecks().AddCheck(name: "Hangfire"); - return builder; - } - - public static IMachineBuilder AddBackgroundJobServer( - this IMachineBuilder builder, - IEnumerable? engineTypes = null - ) - { - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; - var queues = new List(); - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - queues.Add("smt_transfer"); - break; - case TranslationEngineType.Nmt: - builder.AddClearMLService(); - queues.Add("nmt"); - break; - } - } - - builder.Services.AddHangfireServer(o => - { - o.Queues = queues.ToArray(); - }); - return builder; - } - - public static IMachineBuilder AddMemoryDataAccess(this IMachineBuilder builder) - { - builder.Services.AddMemoryDataAccess(o => - { - o.AddRepository(); - o.AddRepository(); - o.AddRepository(); - }); - - return builder; - } - - public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, string? connectionString = null) - { - connectionString ??= builder.Configuration.GetConnectionString("Mongo"); - builder.Services.AddMongoDataAccess( - connectionString, - "SIL.Machine.AspNetCore.Models", - o => - { - o.AddRepository( - "translation_engines", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.EngineId) - ) - ) - ); - o.AddRepository( - "locks", - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("writerLock._id")) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("readerLocks._id")) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("writerQueue._id")) - ); - } - ); - o.AddRepository( - "train_segment_pairs", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) - ) - ) - ); - } - ); - builder.Services.AddHealthChecks().AddMongoDb(connectionString, name: "Mongo"); - - return builder; - } - - public static IMachineBuilder AddServalPlatformService( - this IMachineBuilder builder, - string? connectionString = null - ) - { - builder.Services.AddScoped(); - builder.Services - .AddGrpcClient(o => - { - o.Address = new Uri(connectionString ?? builder.Configuration.GetConnectionString("Serval")); - }) - .ConfigureChannel(o => - { - o.MaxRetryAttempts = null; - o.ServiceConfig = new ServiceConfig - { - MethodConfigs = - { - new MethodConfig - { - Names = { MethodName.Default }, - RetryPolicy = new RetryPolicy - { - MaxAttempts = 10, - InitialBackoff = TimeSpan.FromSeconds(1), - MaxBackoff = TimeSpan.FromSeconds(5), - BackoffMultiplier = 1.5, - RetryableStatusCodes = { StatusCode.Unavailable } - } - }, - new MethodConfig - { - Names = - { - new MethodName - { - Service = "serval.translation.v1.TranslationPlatformApi", - Method = "UpdateBuildStatus" - } - } - }, - } - }; - }); - - return builder; - } - - public static IMachineBuilder AddServalTranslationEngineService( - this IMachineBuilder builder, - string? connectionString = null, - IEnumerable? engineTypes = null - ) - { - builder.Services.AddGrpc(options => options.Interceptors.Add()); - builder.AddServalPlatformService(connectionString ?? builder.Configuration.GetConnectionString("Serval")); - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.Services.AddSingleton(); - builder.Services.AddHostedService(); - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - builder.Services.AddScoped(); - break; - case TranslationEngineType.Nmt: - builder.AddClearMLService(); - builder.Services.AddScoped(); - break; - } - } - builder.Services.AddGrpcHealthChecks(); - - return builder; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs b/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs deleted file mode 100644 index 739a72924..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs +++ /dev/null @@ -1,317 +0,0 @@ -using Microsoft.AspNetCore.Http; -using Serval.Translation.V1; - -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMachineBuilderExtensions -{ - public static IMachineBuilder AddServiceOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddServiceOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddClearMLNmtEngineOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddClearMLNmtEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddThotSmtModel( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder.AddThotSmtModel(); - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder.AddThotSmtModel(); - } - - public static IMachineBuilder AddTransferEngine(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddUnigramTruecaser(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddClearMLService(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); - - // workaround register satisfying the interface and as a hosted service. - builder.Services.AddSingleton(); - builder.Services.AddHostedService(p => p.GetRequiredService()); - - builder.Services - .AddHttpClient() - .AddTransientHttpErrorPolicy( - b => b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) - ); - - return builder; - } - - public static IMachineBuilder AddMongoBackgroundJobClient( - this IMachineBuilder builder, - string? connectionString = null - ) - { - builder.Services.AddHangfire( - c => - c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) - .UseSimpleAssemblyNameTypeSerializer() - .UseRecommendedSerializerSettings() - .UseMongoStorage( - connectionString ?? builder.Configuration.GetConnectionString("Hangfire"), - new MongoStorageOptions - { - MigrationOptions = new MongoMigrationOptions - { - MigrationStrategy = new MigrateMongoMigrationStrategy(), - BackupStrategy = new CollectionMongoBackupStrategy() - }, - CheckConnection = true, - CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, - } - ) - ); - builder.Services.AddHealthChecks().AddCheck(name: "Hangfire"); - return builder; - } - - public static IMachineBuilder AddBackgroundJobServer( - this IMachineBuilder builder, - IEnumerable? engineTypes = null - ) - { - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; - var queues = new List(); - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - queues.Add("smt_transfer"); - break; - case TranslationEngineType.Nmt: - builder.AddClearMLService(); - queues.Add("nmt"); - break; - } - } - - builder.Services.AddHangfireServer(o => - { - o.Queues = queues.ToArray(); - }); - return builder; - } - - public static IMachineBuilder AddMemoryDataAccess(this IMachineBuilder builder) - { - builder.Services.AddMemoryDataAccess(o => - { - o.AddRepository(); - o.AddRepository(); - o.AddRepository(); - }); - - return builder; - } - - public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, string? connectionString = null) - { - connectionString ??= builder.Configuration.GetConnectionString("Mongo"); - builder.Services.AddMongoDataAccess( - connectionString, - "SIL.Machine.AspNetCore.Models", - o => - { - o.AddRepository( - "translation_engines", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.EngineId) - ) - ) - ); - o.AddRepository( - "locks", - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("writerLock._id")) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("readerLocks._id")) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("writerQueue._id")) - ); - } - ); - o.AddRepository( - "train_segment_pairs", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) - ) - ) - ); - } - ); - builder.Services.AddHealthChecks().AddMongoDb(connectionString, name: "Mongo"); - - return builder; - } - - public static IMachineBuilder AddServalPlatformService( - this IMachineBuilder builder, - string? connectionString = null - ) - { - builder.Services.AddScoped(); - builder.Services - .AddGrpcClient(o => - { - o.Address = new Uri(connectionString ?? builder.Configuration.GetConnectionString("Serval")); - }) - .ConfigureChannel(o => - { - o.MaxRetryAttempts = null; - o.ServiceConfig = new ServiceConfig - { - MethodConfigs = - { - new MethodConfig - { - Names = { MethodName.Default }, - RetryPolicy = new Grpc.Net.Client.Configuration.RetryPolicy - { - MaxAttempts = 10, - InitialBackoff = TimeSpan.FromSeconds(1), - MaxBackoff = TimeSpan.FromSeconds(5), - BackoffMultiplier = 1.5, - RetryableStatusCodes = { StatusCode.Unavailable } - } - }, - new MethodConfig - { - Names = - { - new MethodName - { - Service = "serval.translation.v1.TranslationPlatformApi", - Method = "UpdateBuildStatus" - } - } - }, - } - }; - }); - - return builder; - } - - public static IMachineBuilder AddServalTranslationEngineService( - this IMachineBuilder builder, - string? connectionString = null, - IEnumerable? engineTypes = null - ) - { - builder.Services.AddGrpc(options => options.Interceptors.Add()); - builder.AddServalPlatformService(connectionString ?? builder.Configuration.GetConnectionString("Serval")); - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.Services.AddSingleton(); - builder.Services.AddHostedService(); - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - builder.Services.AddScoped(); - break; - case TranslationEngineType.Nmt: - builder.AddClearMLService(); - builder.Services.AddScoped(); - break; - } - } - builder.Services.AddGrpcHealthChecks(); - - return builder; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs deleted file mode 100644 index 9e7eed761..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs +++ /dev/null @@ -1,343 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await sourceTrainWriter.BaseStream.DisposeAsync(); - - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs deleted file mode 100644 index e41df89d3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs +++ /dev/null @@ -1,342 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await targetTrainWriter.BaseStream.DisposeAsync(); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs deleted file mode 100644 index e41df89d3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs +++ /dev/null @@ -1,342 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await targetTrainWriter.BaseStream.DisposeAsync(); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs deleted file mode 100644 index e41df89d3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs +++ /dev/null @@ -1,342 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await targetTrainWriter.BaseStream.DisposeAsync(); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs deleted file mode 100644 index e41df89d3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs +++ /dev/null @@ -1,342 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await targetTrainWriter.BaseStream.DisposeAsync(); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs deleted file mode 100644 index bf3981d14..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs +++ /dev/null @@ -1,342 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await targetTrainWriter.BaseStream.DisposeAsync(); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs deleted file mode 100644 index a26cda901..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs deleted file mode 100644 index 42eb064e1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs +++ /dev/null @@ -1,51 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineService : TranslationEngineServiceBase -{ - private readonly IClearMLService _clearMLService; - - public ClearMLNmtEngineService( - IBackgroundJobClient jobClient, - IPlatformService platformService, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IRepository engines, - IClearMLService clearMLService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _clearMLService = clearMLService; - } - - public override TranslationEngineType Type => TranslationEngineType.Nmt; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - await _clearMLService.CreateProjectAsync(engineId, engineName, cancellationToken: CancellationToken.None); - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - string? projectId = await _clearMLService.GetProjectIdAsync(engineId, CancellationToken.None); - if (projectId is not null) - await _clearMLService.DeleteProjectAsync(projectId, CancellationToken.None); - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs deleted file mode 100644 index 42eb064e1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs +++ /dev/null @@ -1,51 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineService : TranslationEngineServiceBase -{ - private readonly IClearMLService _clearMLService; - - public ClearMLNmtEngineService( - IBackgroundJobClient jobClient, - IPlatformService platformService, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IRepository engines, - IClearMLService clearMLService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _clearMLService = clearMLService; - } - - public override TranslationEngineType Type => TranslationEngineType.Nmt; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - await _clearMLService.CreateProjectAsync(engineId, engineName, cancellationToken: CancellationToken.None); - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - string? projectId = await _clearMLService.GetProjectIdAsync(engineId, CancellationToken.None); - if (projectId is not null) - await _clearMLService.DeleteProjectAsync(projectId, CancellationToken.None); - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs deleted file mode 100644 index 42eb064e1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs +++ /dev/null @@ -1,51 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineService : TranslationEngineServiceBase -{ - private readonly IClearMLService _clearMLService; - - public ClearMLNmtEngineService( - IBackgroundJobClient jobClient, - IPlatformService platformService, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IRepository engines, - IClearMLService clearMLService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _clearMLService = clearMLService; - } - - public override TranslationEngineType Type => TranslationEngineType.Nmt; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - await _clearMLService.CreateProjectAsync(engineId, engineName, cancellationToken: CancellationToken.None); - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - string? projectId = await _clearMLService.GetProjectIdAsync(engineId, CancellationToken.None); - if (projectId is not null) - await _clearMLService.DeleteProjectAsync(projectId, CancellationToken.None); - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs deleted file mode 100644 index 56bbc9829..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs +++ /dev/null @@ -1,95 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - return Task.FromResult( - new BufferedStream(new S3WriteStream(_client, objectId, _bucketName), 1024 * 1024 * 100) - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs deleted file mode 100644 index 56bbc9829..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs +++ /dev/null @@ -1,95 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - return Task.FromResult( - new BufferedStream(new S3WriteStream(_client, objectId, _bucketName), 1024 * 1024 * 100) - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs deleted file mode 100644 index aaf4a3f52..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs +++ /dev/null @@ -1,97 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - 1024 * 1024 * 5 - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs deleted file mode 100644 index 66204bcaa..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs +++ /dev/null @@ -1,97 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - 1024 * 1024 * 5)); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs deleted file mode 100644 index 413860de7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs +++ /dev/null @@ -1,98 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - 1024 * 1024 * 5 - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs deleted file mode 100644 index 413860de7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs +++ /dev/null @@ -1,98 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - 1024 * 1024 * 5 - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs deleted file mode 100644 index 1eeacbc76..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs +++ /dev/null @@ -1,98 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs deleted file mode 100644 index 1eeacbc76..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs +++ /dev/null @@ -1,98 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs deleted file mode 100644 index 2ae672c3c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs deleted file mode 100644 index 0619ee785..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs deleted file mode 100644 index b5eb883cb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - S3WriteStream.FiveMB, - _loggerFactory - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs deleted file mode 100644 index 9058da211..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId, _loggerFactory), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs deleted file mode 100644 index 9058da211..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId, _loggerFactory), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs deleted file mode 100644 index ff8563a42..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId, _loggerFactory), - S3WriteStream.MaxPartSize - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs deleted file mode 100644 index b2d5808f0..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs +++ /dev/null @@ -1,76 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _bucketName; - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _length = 0; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs deleted file mode 100644 index e27f3416a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs +++ /dev/null @@ -1,76 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services;S3Wri - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _bucketName; - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _length = 0; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs deleted file mode 100644 index b2d5808f0..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs +++ /dev/null @@ -1,76 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _bucketName; - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _length = 0; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs deleted file mode 100644 index a4284a6fe..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs +++ /dev/null @@ -1,77 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _bucketName; - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _length = 0; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs deleted file mode 100644 index effbfca90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs +++ /dev/null @@ -1,79 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs deleted file mode 100644 index 1fcc576e6..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs +++ /dev/null @@ -1,78 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs deleted file mode 100644 index 213ddd80a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs +++ /dev/null @@ -1,78 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs deleted file mode 100644 index 24f757833..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs +++ /dev/null @@ -1,87 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - FilePosition = _length - }; - _length += count; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs deleted file mode 100644 index 04454309b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs +++ /dev/null @@ -1,75 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - FilePosition = _length - }; - _length += count; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs deleted file mode 100644 index 46a493f7f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - FilePosition = _length - }; - _length += count; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs deleted file mode 100644 index 46a493f7f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - FilePosition = _length - }; - _length += count; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs deleted file mode 100644 index a5cdde5c5..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs +++ /dev/null @@ -1,106 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber - }; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs deleted file mode 100644 index 16f137a5a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber - }; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs deleted file mode 100644 index 7d72d7d4d..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - Stream = ms - }; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs deleted file mode 100644 index 21f06731c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs deleted file mode 100644 index 21f06731c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs deleted file mode 100644 index c551af600..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs +++ /dev/null @@ -1,109 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - UploadPartResponse response = await _client.UploadPartAsync(request); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs deleted file mode 100644 index 859618376..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs +++ /dev/null @@ -1,110 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - UploadPartResponse response = await _client.UploadPartAsync(request); - if(response.HttpStatusCode != HttpStatusCode.OK) throw new HttpRequestException($"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}") - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs deleted file mode 100644 index 38f8cf6d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs +++ /dev/null @@ -1,113 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs deleted file mode 100644 index 4e5370496..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs +++ /dev/null @@ -1,116 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - uploadRequest.StreamTransferProgress += new EventHandler( - UploadPartProgressEventCallback - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs deleted file mode 100644 index e89e96826..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs +++ /dev/null @@ -1,114 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler((_, e) => { }); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs deleted file mode 100644 index 7be03bf75..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs +++ /dev/null @@ -1,113 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler((_, e) => { }); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs deleted file mode 100644 index 785cf5858..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs +++ /dev/null @@ -1,114 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler((_, e) => { }); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs deleted file mode 100644 index 64918822a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs +++ /dev/null @@ -1,115 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = (new LoggerFactory()).CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler((_, e) => { }); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs deleted file mode 100644 index 3a05d9ab0..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs +++ /dev/null @@ -1,115 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler((_, e) => { }); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs deleted file mode 100644 index 2d1c7f47a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs deleted file mode 100644 index 2d1c7f47a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs deleted file mode 100644 index 7c5630f4b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs deleted file mode 100644 index 7c5630f4b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs deleted file mode 100644 index 52ecc2ef3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs deleted file mode 100644 index 52ecc2ef3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs deleted file mode 100644 index cb6f6edb5..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs deleted file mode 100644 index d6bc7c0aa..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs deleted file mode 100644 index 81005c673..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs deleted file mode 100644 index bff5aa1d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs deleted file mode 100644 index ac7aa225c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs +++ /dev/null @@ -1,124 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs deleted file mode 100644 index 5f5b59245..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs deleted file mode 100644 index 530e928ca..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs +++ /dev/null @@ -1,124 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs deleted file mode 100644 index 530e928ca..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs +++ /dev/null @@ -1,124 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs deleted file mode 100644 index 7e13f6a62..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs deleted file mode 100644 index 104dc7116..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs deleted file mode 100644 index 104dc7116..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs deleted file mode 100644 index a97937131..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs +++ /dev/null @@ -1,126 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs deleted file mode 100644 index a97937131..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs +++ /dev/null @@ -1,126 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs deleted file mode 100644 index a97937131..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs +++ /dev/null @@ -1,126 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs deleted file mode 100644 index 1024e4aec..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs +++ /dev/null @@ -1,127 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) { } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs deleted file mode 100644 index 4ef04e880..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs +++ /dev/null @@ -1,151 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs deleted file mode 100644 index e4c5eb276..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs +++ /dev/null @@ -1,151 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client.CompleteMultipartUpload(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs deleted file mode 100644 index 9a03bb2e4..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs deleted file mode 100644 index 584a0e09c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs +++ /dev/null @@ -1,153 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs deleted file mode 100644 index 9b4f9cf96..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public static final int fiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs deleted file mode 100644 index d7b0b6982..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public final static int fiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs deleted file mode 100644 index a47284fb5..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public static const int fiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs deleted file mode 100644 index 50a58fafe..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int fiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs deleted file mode 100644 index c13ee9bc2..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FIVE_MB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs deleted file mode 100644 index c13ee9bc2..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FIVE_MB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs deleted file mode 100644 index 20cee4645..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMb = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs deleted file mode 100644 index 262f9a6c3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMbB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs deleted file mode 100644 index 9ea77dfee..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs deleted file mode 100644 index 9ea77dfee..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs deleted file mode 100644 index 6d25881bb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs deleted file mode 100644 index 6d25881bb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs deleted file mode 100644 index 95dd7899d..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs +++ /dev/null @@ -1,161 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs deleted file mode 100644 index 001301acb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs +++ /dev/null @@ -1,161 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs deleted file mode 100644 index 001301acb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs +++ /dev/null @@ -1,161 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs deleted file mode 100644 index 7fc4c6f7a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs +++ /dev/null @@ -1,162 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs deleted file mode 100644 index afd96e3f9..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs deleted file mode 100644 index afd96e3f9..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs deleted file mode 100644 index 339ca3a26..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs deleted file mode 100644 index 339ca3a26..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs deleted file mode 100644 index fc2173053..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int MaxPartSize = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = MaxPartSize - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs deleted file mode 100644 index fc2173053..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int MaxPartSize = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = MaxPartSize - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs deleted file mode 100644 index fc2173053..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int MaxPartSize = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = MaxPartSize - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs deleted file mode 100644 index 526cf95fd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs +++ /dev/null @@ -1,92 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - - public SharedFileService(IOptions? options = null) - { - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs deleted file mode 100644 index 526cf95fd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs +++ /dev/null @@ -1,92 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - - public SharedFileService(IOptions? options = null) - { - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs deleted file mode 100644 index a7a778353..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs +++ /dev/null @@ -1,92 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - - public SharedFileService(IOptions? options = null) - { - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - }} - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs deleted file mode 100644 index 526cf95fd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs +++ /dev/null @@ -1,92 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - - public SharedFileService(IOptions? options = null) - { - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs deleted file mode 100644 index c7eb4f847..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs +++ /dev/null @@ -1,95 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) - { - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs deleted file mode 100644 index 53777aeaa..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs +++ /dev/null @@ -1,97 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs deleted file mode 100644 index c9f4aa6b1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs +++ /dev/null @@ -1,98 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs deleted file mode 100644 index c9f4aa6b1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs +++ /dev/null @@ -1,98 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs deleted file mode 100644 index 5804584c1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs +++ /dev/null @@ -1,98 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs deleted file mode 100644 index 5804584c1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs +++ /dev/null @@ -1,98 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs deleted file mode 100644 index cef73bbeb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs +++ /dev/null @@ -1,96 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs deleted file mode 100644 index cef73bbeb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs +++ /dev/null @@ -1,96 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs deleted file mode 100644 index e7619bc63..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs +++ /dev/null @@ -1,181 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs deleted file mode 100644 index 804a21c2e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs +++ /dev/null @@ -1,183 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - if (state.CurrentBuildRevision == -1) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs deleted file mode 100644 index 804a21c2e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs +++ /dev/null @@ -1,183 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - if (state.CurrentBuildRevision == -1) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs deleted file mode 100644 index 804a21c2e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs +++ /dev/null @@ -1,183 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - if (state.CurrentBuildRevision == -1) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs deleted file mode 100644 index 68d6dc96e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs +++ /dev/null @@ -1,183 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - if (state.CurrentBuildRevision == -1) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs deleted file mode 100644 index 38b27e746..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if (await Engines.GetAsync(e => e.Id == engineId && e.BuildId)) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs deleted file mode 100644 index 4c6134aa9..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - ( - await Engines.GetAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs deleted file mode 100644 index cf92eef1a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - ( - await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs deleted file mode 100644 index cf92eef1a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - ( - await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs deleted file mode 100644 index 4672cf290..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - !( - await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs deleted file mode 100644 index 4672cf290..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - !( - await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs deleted file mode 100644 index f75ec89f0..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs +++ /dev/null @@ -1,190 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs deleted file mode 100644 index f75ec89f0..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs +++ /dev/null @@ -1,190 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs deleted file mode 100644 index 22727b2d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs +++ /dev/null @@ -1,189 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs deleted file mode 100644 index 22727b2d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs +++ /dev/null @@ -1,189 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs deleted file mode 100644 index 22727b2d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs +++ /dev/null @@ -1,189 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs deleted file mode 100644 index 22727b2d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs +++ /dev/null @@ -1,189 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs deleted file mode 100644 index f5d00996f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs +++ /dev/null @@ -1,190 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - if ( - !( - - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs deleted file mode 100644 index dd3aa1037..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs +++ /dev/null @@ -1,192 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach(var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if ( - !( - - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs deleted file mode 100644 index b773a87c2..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!(engines).Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs deleted file mode 100644 index 46ad52609..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs +++ /dev/null @@ -1,189 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Console.WriteLine("|||||||||||||||||||"); - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs deleted file mode 100644 index 9991f0c87..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs +++ /dev/null @@ -1,191 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Console.WriteLine("|||||||||||||||||||"); - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - Console.WriteLine("|||||||||||||||||||"); - - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs deleted file mode 100644 index 3574fa79f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs +++ /dev/null @@ -1,190 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Console.WriteLine("|||||||||||||||||||"); - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - Console.WriteLine("|||||||||||||||||||"); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs deleted file mode 100644 index 47ce83b1f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs +++ /dev/null @@ -1,190 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Console.WriteLine("|||||||||||||||||||"); - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - Console.WriteLine("|||||||||||||||||||"); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs deleted file mode 100644 index 234a20433..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Console.WriteLine("|||||||||||||||||||"); - IEnumerable engines = await Engines.GetAllAsync(e => e.Id == engineId); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - Console.WriteLine("|||||||||||||||||||"); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs deleted file mode 100644 index 92d3d3d1e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs +++ /dev/null @@ -1,185 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs deleted file mode 100644 index 26186680c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs deleted file mode 100644 index eaa649a10..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs +++ /dev/null @@ -1,185 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs deleted file mode 100644 index 76069459f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildRevision == 0 || engine.BuildState != BuildState.None || engine.Revision == 0) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs deleted file mode 100644 index 76069459f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildRevision == 0 || engine.BuildState != BuildState.None || engine.Revision == 0) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs deleted file mode 100644 index 76069459f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildRevision == 0 || engine.BuildState != BuildState.None || engine.Revision == 0) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs deleted file mode 100644 index b73ba6ead..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.Revision == 0) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs deleted file mode 100644 index b73ba6ead..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.Revision == 0) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs deleted file mode 100644 index ebb403ba9..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.Revision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs deleted file mode 100644 index 296b73779..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs deleted file mode 100644 index 296b73779..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs deleted file mode 100644 index 296b73779..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs deleted file mode 100644 index 296b73779..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs deleted file mode 100644 index 3ab3f9d3f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs deleted file mode 100644 index e4be91c25..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs deleted file mode 100644 index 91a738627..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs +++ /dev/null @@ -1,175 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs deleted file mode 100644 index e4eba1d38..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs +++ /dev/null @@ -1,174 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs deleted file mode 100644 index b4dde802b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs +++ /dev/null @@ -1,175 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs deleted file mode 100644 index 0eae7ff8b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs +++ /dev/null @@ -1,174 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs deleted file mode 100644 index 6bc79f5bd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs deleted file mode 100644 index 6bc79f5bd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs deleted file mode 100644 index 8538f7d87..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await G(engineetBuiltEngineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs deleted file mode 100644 index 6bc79f5bd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs deleted file mode 100644 index 67a1023da..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngine(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs deleted file mode 100644 index 2cfccdc7e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs deleted file mode 100644 index 2cfccdc7e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs deleted file mode 100644 index 0f3d710af..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs +++ /dev/null @@ -1,212 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs deleted file mode 100644 index 41ff71d4b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs +++ /dev/null @@ -1,219 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs deleted file mode 100644 index d7acd2bfd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs +++ /dev/null @@ -1,219 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs deleted file mode 100644 index 2d7e5e928..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs +++ /dev/null @@ -1,227 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs deleted file mode 100644 index 2d7e5e928..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs +++ /dev/null @@ -1,227 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs deleted file mode 100644 index 2d7e5e928..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs +++ /dev/null @@ -1,227 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs deleted file mode 100644 index 2d7e5e928..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs +++ /dev/null @@ -1,227 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs deleted file mode 100644 index 8c2e0cbc3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs +++ /dev/null @@ -1,230 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt( - string engineId, - CancellationToken cancellationToken - ) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs deleted file mode 100644 index f310e85f3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs +++ /dev/null @@ -1,227 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs deleted file mode 100644 index dc702f8de..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs deleted file mode 100644 index dc702f8de..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs deleted file mode 100644 index d6511bf11..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs deleted file mode 100644 index dc702f8de..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs deleted file mode 100644 index dc702f8de..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs deleted file mode 100644 index 21868db71..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs +++ /dev/null @@ -1,47 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.S3.Transfer; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs deleted file mode 100644 index 0b07fae03..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs +++ /dev/null @@ -1,47 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.Runtime; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs deleted file mode 100644 index a808510e6..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs +++ /dev/null @@ -1,49 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.Runtime; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -using Nito.AsyncEx.Synchronous; - -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs deleted file mode 100644 index 981c59ef5..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs +++ /dev/null @@ -1,48 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.Runtime; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -using Nito.AsyncEx.Synchronous; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs deleted file mode 100644 index 8544d1e96..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs +++ /dev/null @@ -1,48 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.Runtime; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -global using Nito.AsyncEx.Synchronous; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs deleted file mode 100644 index 8544d1e96..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs +++ /dev/null @@ -1,48 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.Runtime; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -global using Nito.AsyncEx.Synchronous; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs b/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs deleted file mode 100644 index f9d01a7d6..000000000 --- a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs +++ /dev/null @@ -1,192 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class ClearMLNmtEngineServiceTests -{ - [Test] - public async Task CancelBuildAsync() - { - using var env = new TestEnvironment(); - env.ClearMLService - .CreateTaskAsync( - Arg.Any(), - "project1", - "engine1", - "es", - "en", - "memory:///", - Arg.Any() - ) - .Returns(Task.FromResult("task1")); - var task = new ClearMLTask - { - Id = "task1", - Project = new ClearMLProject { Id = "project1" }, - Status = ClearMLTaskStatus.InProgress - }; - bool first = true; - env.ClearMLService - .GetTaskByNameAsync(Arg.Any(), Arg.Any()) - .Returns(x => - { - if (first) - { - first = false; - return Task.FromResult(null); - } - return Task.FromResult(task); - }); - env.ClearMLService - .GetTaskByIdAsync("task1", Arg.Any()) - .Returns(Task.FromResult(task)); - await env.Service.StartBuildAsync("engine1", "build1", Array.Empty()); - await env.WaitForBuildToStartAsync(); - TranslationEngine engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildState, Is.EqualTo(BuildState.Active)); - await env.Service.CancelBuildAsync("engine1"); - await env.WaitForBuildToFinishAsync(); - engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildState, Is.EqualTo(BuildState.None)); - await env.ClearMLService.Received().StopTaskAsync("task1", Arg.Any()); - } - - private class TestEnvironment : DisposableBase - { - private readonly MemoryStorage _memoryStorage; - private readonly BackgroundJobClient _jobClient; - private BackgroundJobServer _jobServer; - private readonly IDistributedReaderWriterLockFactory _lockFactory; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - - public TestEnvironment() - { - Engines = new MemoryRepository(); - Engines.Add( - new TranslationEngine - { - Id = "engine1", - EngineId = "engine1", - SourceLanguage = "es", - TargetLanguage = "en" - } - ); - EngineOptions = new SmtTransferEngineOptions(); - _memoryStorage = new MemoryStorage(); - _jobClient = new BackgroundJobClient(_memoryStorage); - PlatformService = Substitute.For(); - ClearMLService = Substitute.For(); - ClearMLService - .GetProjectIdAsync(Arg.Any(), Arg.Any()) - .Returns(Task.FromResult("project1")); - _lockFactory = new DistributedReaderWriterLockFactory( - new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), - new MemoryRepository(), - new ObjectIdGenerator() - ); - _sharedFileService = new SharedFileService(); - _options = Substitute.For>(); - _options.CurrentValue.Returns( - new ClearMLNmtEngineOptions { BuildPollingTimeout = TimeSpan.FromMilliseconds(50) } - ); - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - public ClearMLNmtEngineService Service { get; private set; } - public MemoryRepository Engines { get; } - public SmtTransferEngineOptions EngineOptions { get; } - public IPlatformService PlatformService { get; } - public IClearMLService ClearMLService { get; } - - public void StopServer() - { - _jobServer.Dispose(); - } - - public void StartServer() - { - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - private BackgroundJobServer CreateJobServer() - { - var jobServerOptions = new BackgroundJobServerOptions - { - Activator = new EnvActivator(this), - Queues = new[] { "nmt" }, - CancellationCheckInterval = TimeSpan.FromMilliseconds(100), - }; - return new BackgroundJobServer(jobServerOptions, _memoryStorage); - } - - private ClearMLNmtEngineService CreateService() - { - return new ClearMLNmtEngineService( - _jobClient, - PlatformService, - _lockFactory, - new MemoryDataAccessContext(), - Engines, - ClearMLService - ); - } - - public Task WaitForBuildToFinishAsync() - { - return WaitForBuildState(e => e.BuildState is BuildState.None); - } - - public Task WaitForBuildToStartAsync() - { - return WaitForBuildState(e => e.BuildState is BuildState.Active); - } - - private async Task WaitForBuildState(Func predicate) - { - using ISubscription subscription = await Engines.SubscribeAsync( - e => e.EngineId == "engine1" - ); - while (true) - { - TranslationEngine? build = subscription.Change.Entity; - if (build is not null && predicate(build)) - break; - await subscription.WaitForChangeAsync(); - } - } - - protected override void DisposeManagedResources() - { - _jobServer.Dispose(); - } - - private class EnvActivator : JobActivator - { - private readonly TestEnvironment _env; - - public EnvActivator(TestEnvironment env) - { - _env = env; - } - - public override object ActivateJob(Type jobType) - { - if (jobType == typeof(ClearMLNmtEngineBuildJob)) - { - return new ClearMLNmtEngineBuildJob( - _env.PlatformService, - _env.Engines, - Substitute.For>(), - _env.ClearMLService, - _env._sharedFileService, - _env._options, - Substitute.For() - ); - } - return base.ActivateJob(jobType); - } - } - } -} diff --git a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs b/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs deleted file mode 100644 index 614dabc52..000000000 --- a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs +++ /dev/null @@ -1,192 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class ClearMLNmtEngineServiceTests -{ - [Test] - public async Task CancelBuildAsync() - { - using var env = new TestEnvironment(); - env.ClearMLService - .CreateTaskAsync( - Arg.Any(), - "project1", - "engine1", - "es", - "en", - "memory:///", - Arg.Any() - ) - .Returns(Task.FromResult("task1")); - var task = new ClearMLTask - { - Id = "task1", - Project = new ClearMLProject { Id = "project1" }, - Status = ClearMLTaskStatus.InProgress - }; - bool first = true; - env.ClearMLService - .GetTaskByNameAsync(Arg.Any(), Arg.Any()) - .Returns(x => - { - if (first) - { - first = false; - return Task.FromResult(null); - } - return Task.FromResult(task); - }); - env.ClearMLService - .GetTaskByIdAsync("task1", Arg.Any()) - .Returns(Task.FromResult(task)); - await env.Service.StartBuildAsync("engine1", "build1", Array.Empty()); - await env.WaitForBuildToStartAsync(); - TranslationEngine engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildState, Is.EqualTo(BuildState.Active)); - await env.Service.CancelBuildAsync("engine1"); - await env.WaitForBuildToFinishAsync(); - engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildState, Is.EqualTo(BuildState.None)); - await env.ClearMLService.Received().StopTaskAsync("task1", Arg.Any()); - } - - private class TestEnvironment : DisposableBase - { - private readonly MemoryStorage _memoryStorage; - private readonly BackgroundJobClient _jobClient; - private BackgroundJobServer _jobServer; - private readonly IDistributedReaderWriterLockFactory _lockFactory; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - - public TestEnvironment() - { - Engines = new MemoryRepository(); - Engines.Add( - new TranslationEngine - { - Id = "engine1", - EngineId = "engine1", - SourceLanguage = "es", - TargetLanguage = "en" - } - ); - EngineOptions = new SmtTransferEngineOptions(); - _memoryStorage = new MemoryStorage(); - _jobClient = new BackgroundJobClient(_memoryStorage); - PlatformService = Substitute.For(); - ClearMLService = Substitute.For(); - ClearMLService - .GetProjectIdAsync(Arg.Any(), Arg.Any()) - .Returns(Task.FromResult("project1")); - _lockFactory = new DistributedReaderWriterLockFactory( - new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), - new MemoryRepository(), - new ObjectIdGenerator() - ); - _sharedFileService = new SharedFileService(Substitute.For()); - _options = Substitute.For>(); - _options.CurrentValue.Returns( - new ClearMLNmtEngineOptions { BuildPollingTimeout = TimeSpan.FromMilliseconds(50) } - ); - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - public ClearMLNmtEngineService Service { get; private set; } - public MemoryRepository Engines { get; } - public SmtTransferEngineOptions EngineOptions { get; } - public IPlatformService PlatformService { get; } - public IClearMLService ClearMLService { get; } - - public void StopServer() - { - _jobServer.Dispose(); - } - - public void StartServer() - { - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - private BackgroundJobServer CreateJobServer() - { - var jobServerOptions = new BackgroundJobServerOptions - { - Activator = new EnvActivator(this), - Queues = new[] { "nmt" }, - CancellationCheckInterval = TimeSpan.FromMilliseconds(100), - }; - return new BackgroundJobServer(jobServerOptions, _memoryStorage); - } - - private ClearMLNmtEngineService CreateService() - { - return new ClearMLNmtEngineService( - _jobClient, - PlatformService, - _lockFactory, - new MemoryDataAccessContext(), - Engines, - ClearMLService - ); - } - - public Task WaitForBuildToFinishAsync() - { - return WaitForBuildState(e => e.BuildState is BuildState.None); - } - - public Task WaitForBuildToStartAsync() - { - return WaitForBuildState(e => e.BuildState is BuildState.Active); - } - - private async Task WaitForBuildState(Func predicate) - { - using ISubscription subscription = await Engines.SubscribeAsync( - e => e.EngineId == "engine1" - ); - while (true) - { - TranslationEngine? build = subscription.Change.Entity; - if (build is not null && predicate(build)) - break; - await subscription.WaitForChangeAsync(); - } - } - - protected override void DisposeManagedResources() - { - _jobServer.Dispose(); - } - - private class EnvActivator : JobActivator - { - private readonly TestEnvironment _env; - - public EnvActivator(TestEnvironment env) - { - _env = env; - } - - public override object ActivateJob(Type jobType) - { - if (jobType == typeof(ClearMLNmtEngineBuildJob)) - { - return new ClearMLNmtEngineBuildJob( - _env.PlatformService, - _env.Engines, - Substitute.For>(), - _env.ClearMLService, - _env._sharedFileService, - _env._options, - Substitute.For() - ); - } - return base.ActivateJob(jobType); - } - } - } -} From eda3591f5239871fbdcfd68effc9be8439bea206 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Fri, 1 Sep 2023 09:55:49 -0400 Subject: [PATCH 07/10] Fixes https://github.com/sillsdev/serval/issues/100 -- ECL --- .github/workflows/ci-e2e.yml | 64 ++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/ci-e2e.yml diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml new file mode 100644 index 000000000..cd107444d --- /dev/null +++ b/.github/workflows/ci-e2e.yml @@ -0,0 +1,64 @@ +name: "CI Build: E2E tests" + +on: + push: + branches: ["run_e2etests_on_release_#100"] + release: + types: [published] + workflow_dispatch: + +jobs: + build: + name: Build + runs-on: ubuntu-latest + timeout-minutes: 15 + + env: + SERVAL_CLIENT_ID: ${{ secrets.SERVAL_CLIENT_ID }} + SERVAL_CLIENT_SECRET: ${{ secrets.SERVAL_CLIENT_SECRET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + ClearML_AccessKey: ${{ secrets.CLEARML_ACCESSKEY }} + ClearML_SecretKey: ${{ secrets.CLEARML_SECRETKEY }} + SERVAL_HOST_URL: http://localhost + SERVAL_AUTH_URL: https://sil-appbuilder.auth0.com + + steps: + - uses: actions/checkout@v3 + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + with: + dotnet-version: 6.0.x + + - name: Get Machine + run: dotnet build && cd .. && git clone https://github.com/sillsdev/serval.git && cd serval && dotnet build + + - name: Restore dotnet tools + run: dotnet tool restore + + - name: Restore dependencies + run: dotnet restore + + - name: Build + run: dotnet build --no-restore + + - name: Start containers + run: docker compose -f "docker-compose.yml" up -d && sleep 20 #allow time for mongo to start up properly + + - name: Debug network + run: docker ps -a && docker logs --since 10m serval_cntr && docker logs --since 10m echo_cntr && docker logs --since 10m machine-engine-cntr && docker logs --since 10m serval-mongo-1 && docker logs --since 10m machine-job-cntr + + - name: Pre-Test + run: sudo mkdir -p /var/lib/serval && sudo chmod 777 /var/lib/serval + + - name: Test + run: dotnet test --no-build --verbosity normal --filter "TestCategory!=slow&TestCategory=E2E" + + - name: Debug network (Post test) + if: ${{ failure() }} + run: docker ps -a && docker logs --since 10m serval_cntr && docker logs --since 10m echo_cntr && docker logs --since 10m machine-engine-cntr && docker logs --since 10m serval-mongo-1 && docker logs --since 10m machine-job-cntr + + - name: Stop containers + if: ${{ success() || failure() }} + run: docker compose down From 67becc41bed2f94f8bca6b72d128f2cc865565fe Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Fri, 1 Sep 2023 10:03:53 -0400 Subject: [PATCH 08/10] Fixed directory issue --ECL --- .github/workflows/ci-e2e.yml | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml index cd107444d..fba5cea23 100644 --- a/.github/workflows/ci-e2e.yml +++ b/.github/workflows/ci-e2e.yml @@ -31,20 +31,11 @@ jobs: with: dotnet-version: 6.0.x - - name: Get Machine + - name: Get Serval run: dotnet build && cd .. && git clone https://github.com/sillsdev/serval.git && cd serval && dotnet build - - name: Restore dotnet tools - run: dotnet tool restore - - - name: Restore dependencies - run: dotnet restore - - - name: Build - run: dotnet build --no-restore - - name: Start containers - run: docker compose -f "docker-compose.yml" up -d && sleep 20 #allow time for mongo to start up properly + run: docker compose -f "../serval/docker-compose.yml" up -d && sleep 20 #allow time for mongo to start up properly - name: Debug network run: docker ps -a && docker logs --since 10m serval_cntr && docker logs --since 10m echo_cntr && docker logs --since 10m machine-engine-cntr && docker logs --since 10m serval-mongo-1 && docker logs --since 10m machine-job-cntr @@ -53,7 +44,7 @@ jobs: run: sudo mkdir -p /var/lib/serval && sudo chmod 777 /var/lib/serval - name: Test - run: dotnet test --no-build --verbosity normal --filter "TestCategory!=slow&TestCategory=E2E" + run: cd ../serval && dotnet test --no-build --verbosity normal --filter "TestCategory!=slow&TestCategory=E2E" - name: Debug network (Post test) if: ${{ failure() }} @@ -61,4 +52,4 @@ jobs: - name: Stop containers if: ${{ success() || failure() }} - run: docker compose down + run: docker compose -f "../serval/docker-compose.yml" down From bb364f31828b26f5487825a4346f6417a7cbe1c7 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Mon, 11 Sep 2023 13:02:29 -0400 Subject: [PATCH 09/10] Empty commit to trigger workflow --ECL --- .history/.gitignore_20230911115750 | 51 +++ .history/.gitignore_20230911115828 | 52 +++ ...MachineBuilderExtensions_20230907084316.cs | 317 ++++++++++++++++ ...MachineBuilderExtensions_20230907133310.cs | 317 ++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907084316.cs | 340 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907095831.cs | 340 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907135725.cs | 343 ++++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907135737.cs | 342 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907135738.cs | 342 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907135739.cs | 342 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907140302.cs | 342 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907140611.cs | 342 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907140709.cs | 340 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907140719.cs | 340 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907140720.cs | 340 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907141110.cs | 340 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907141112.cs | 340 +++++++++++++++++ ...ClearMLNmtEngineBuildJob_20230907141905.cs | 340 +++++++++++++++++ .../ClearMLNmtEngineService_20230906141540.cs | 51 +++ .../ClearMLNmtEngineService_20230911085826.cs | 51 +++ .../ClearMLNmtEngineService_20230911085829.cs | 51 +++ .../Services/S3FileStorage_20230906141540.cs | 95 +++++ .../Services/S3FileStorage_20230907104119.cs | 95 +++++ .../Services/S3FileStorage_20230907130153.cs | 97 +++++ .../Services/S3FileStorage_20230907130157.cs | 97 +++++ .../Services/S3FileStorage_20230907132651.cs | 98 +++++ .../Services/S3FileStorage_20230907132652.cs | 98 +++++ .../Services/S3FileStorage_20230907171652.cs | 98 +++++ .../Services/S3FileStorage_20230907171653.cs | 98 +++++ .../Services/S3FileStorage_20230907171816.cs | 107 ++++++ .../Services/S3FileStorage_20230907171821.cs | 107 ++++++ .../Services/S3FileStorage_20230907171829.cs | 108 ++++++ .../Services/S3FileStorage_20230907171837.cs | 107 ++++++ .../Services/S3FileStorage_20230907171838.cs | 107 ++++++ .../Services/S3FileStorage_20230908125336.cs | 107 ++++++ .../Services/S3WriteStream_20230906141540.cs | 76 ++++ .../Services/S3WriteStream_20230907104130.cs | 76 ++++ .../Services/S3WriteStream_20230907104133.cs | 76 ++++ .../Services/S3WriteStream_20230907130227.cs | 77 ++++ .../Services/S3WriteStream_20230907130326.cs | 79 ++++ .../Services/S3WriteStream_20230907130328.cs | 78 ++++ .../Services/S3WriteStream_20230907130334.cs | 78 ++++ .../Services/S3WriteStream_20230907130427.cs | 87 +++++ .../Services/S3WriteStream_20230907130442.cs | 75 ++++ .../Services/S3WriteStream_20230907130507.cs | 108 ++++++ .../Services/S3WriteStream_20230907130523.cs | 108 ++++++ .../Services/S3WriteStream_20230907130903.cs | 106 ++++++ .../Services/S3WriteStream_20230907130942.cs | 107 ++++++ .../Services/S3WriteStream_20230907130949.cs | 108 ++++++ .../Services/S3WriteStream_20230907131020.cs | 108 ++++++ .../Services/S3WriteStream_20230907131032.cs | 108 ++++++ .../Services/S3WriteStream_20230907131048.cs | 109 ++++++ .../Services/S3WriteStream_20230907131526.cs | 110 ++++++ .../Services/S3WriteStream_20230907131529.cs | 113 ++++++ .../Services/S3WriteStream_20230907131704.cs | 116 ++++++ .../Services/S3WriteStream_20230907131835.cs | 114 ++++++ .../Services/S3WriteStream_20230907131840.cs | 113 ++++++ .../Services/S3WriteStream_20230907131855.cs | 114 ++++++ .../Services/S3WriteStream_20230907131914.cs | 115 ++++++ .../Services/S3WriteStream_20230907131918.cs | 115 ++++++ .../Services/S3WriteStream_20230907132115.cs | 120 ++++++ .../Services/S3WriteStream_20230907132117.cs | 120 ++++++ .../Services/S3WriteStream_20230907132230.cs | 120 ++++++ .../Services/S3WriteStream_20230907132231.cs | 120 ++++++ .../Services/S3WriteStream_20230907132240.cs | 120 ++++++ .../Services/S3WriteStream_20230907132242.cs | 120 ++++++ .../Services/S3WriteStream_20230907132338.cs | 120 ++++++ .../Services/S3WriteStream_20230907132350.cs | 125 +++++++ .../Services/S3WriteStream_20230907132400.cs | 125 +++++++ .../Services/S3WriteStream_20230907132417.cs | 125 +++++++ .../Services/S3WriteStream_20230907132419.cs | 124 +++++++ .../Services/S3WriteStream_20230907132431.cs | 125 +++++++ .../Services/S3WriteStream_20230907132432.cs | 124 +++++++ .../Services/S3WriteStream_20230907132433.cs | 124 +++++++ .../Services/S3WriteStream_20230907132616.cs | 125 +++++++ .../Services/S3WriteStream_20230907132630.cs | 125 +++++++ .../Services/S3WriteStream_20230907132632.cs | 125 +++++++ .../Services/S3WriteStream_20230907132832.cs | 126 +++++++ .../Services/S3WriteStream_20230907132840.cs | 126 +++++++ .../Services/S3WriteStream_20230907133744.cs | 126 +++++++ .../Services/S3WriteStream_20230907140811.cs | 127 +++++++ .../Services/S3WriteStream_20230907140817.cs | 151 ++++++++ .../Services/S3WriteStream_20230907140826.cs | 151 ++++++++ .../Services/S3WriteStream_20230907141040.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141052.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141053.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141054.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141056.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141058.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141059.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141100.cs | 155 ++++++++ .../Services/S3WriteStream_20230907141102.cs | 155 ++++++++ .../Services/S3WriteStream_20230907142255.cs | 153 ++++++++ .../Services/S3WriteStream_20230907171436.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171441.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171450.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171526.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171534.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171604.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171626.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171627.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171628.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171630.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171636.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171638.cs | 155 ++++++++ .../Services/S3WriteStream_20230907171720.cs | 161 ++++++++ .../Services/S3WriteStream_20230907171734.cs | 161 ++++++++ .../Services/S3WriteStream_20230907171736.cs | 161 ++++++++ .../Services/S3WriteStream_20230907172014.cs | 162 +++++++++ .../Services/S3WriteStream_20230907172018.cs | 163 +++++++++ .../Services/S3WriteStream_20230907172019.cs | 163 +++++++++ .../Services/S3WriteStream_20230907172038.cs | 163 +++++++++ .../Services/S3WriteStream_20230907172039.cs | 163 +++++++++ .../Services/S3WriteStream_20230908125336.cs | 163 +++++++++ .../Services/S3WriteStream_20230908125337.cs | 163 +++++++++ .../Services/S3WriteStream_20230908125344.cs | 163 +++++++++ .../SharedFileService_20230906141540.cs | 92 +++++ .../SharedFileService_20230907104153.cs | 92 +++++ .../SharedFileService_20230907134349.cs | 92 +++++ .../SharedFileService_20230907134355.cs | 92 +++++ .../SharedFileService_20230907171909.cs | 95 +++++ .../SharedFileService_20230907171920.cs | 97 +++++ .../SharedFileService_20230907171930.cs | 98 +++++ .../SharedFileService_20230907171931.cs | 98 +++++ .../SharedFileService_20230907171955.cs | 98 +++++ .../SharedFileService_20230907171956.cs | 98 +++++ .../SharedFileService_20230908125406.cs | 96 +++++ .../SharedFileService_20230908125407.cs | 96 +++++ ...SmtTransferEngineService_20230907084301.cs | 181 +++++++++ ...SmtTransferEngineService_20230908161128.cs | 183 ++++++++++ ...SmtTransferEngineService_20230908161129.cs | 183 ++++++++++ ...SmtTransferEngineService_20230908161150.cs | 183 ++++++++++ ...SmtTransferEngineService_20230908161950.cs | 183 ++++++++++ ...SmtTransferEngineService_20230908162047.cs | 184 ++++++++++ ...SmtTransferEngineService_20230908162317.cs | 188 ++++++++++ ...SmtTransferEngineService_20230908162334.cs | 188 ++++++++++ ...SmtTransferEngineService_20230908162336.cs | 188 ++++++++++ ...SmtTransferEngineService_20230908162356.cs | 188 ++++++++++ ...SmtTransferEngineService_20230908162358.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911083245.cs | 190 ++++++++++ ...SmtTransferEngineService_20230911083247.cs | 190 ++++++++++ ...SmtTransferEngineService_20230911083620.cs | 189 ++++++++++ ...SmtTransferEngineService_20230911083625.cs | 189 ++++++++++ ...SmtTransferEngineService_20230911083627.cs | 189 ++++++++++ ...SmtTransferEngineService_20230911083628.cs | 189 ++++++++++ ...SmtTransferEngineService_20230911083939.cs | 190 ++++++++++ ...SmtTransferEngineService_20230911084126.cs | 192 ++++++++++ ...SmtTransferEngineService_20230911084129.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084135.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084136.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084149.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084150.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084151.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084152.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084153.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084154.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084155.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084156.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084157.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084158.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084159.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084200.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084201.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084202.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084203.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084204.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084205.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084231.cs | 189 ++++++++++ ...SmtTransferEngineService_20230911084236.cs | 191 ++++++++++ ...SmtTransferEngineService_20230911084237.cs | 190 ++++++++++ ...SmtTransferEngineService_20230911084410.cs | 190 ++++++++++ ...SmtTransferEngineService_20230911084539.cs | 188 ++++++++++ ...SmtTransferEngineService_20230911084556.cs | 185 ++++++++++ ...SmtTransferEngineService_20230911084557.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084610.cs | 185 ++++++++++ ...SmtTransferEngineService_20230911084658.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084700.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084701.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084711.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084713.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084838.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084843.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084844.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084845.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911084846.cs | 184 ++++++++++ ...SmtTransferEngineService_20230911085524.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085526.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085550.cs | 175 +++++++++ ...SmtTransferEngineService_20230911085551.cs | 174 +++++++++ ...SmtTransferEngineService_20230911085557.cs | 175 +++++++++ ...SmtTransferEngineService_20230911085558.cs | 174 +++++++++ ...SmtTransferEngineService_20230911085704.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085706.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085712.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085757.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085803.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085805.cs | 173 +++++++++ ...SmtTransferEngineService_20230911085809.cs | 173 +++++++++ ...slationEngineServiceBase_20230907084301.cs | 212 +++++++++++ ...slationEngineServiceBase_20230911085214.cs | 219 +++++++++++ ...slationEngineServiceBase_20230911085221.cs | 219 +++++++++++ ...slationEngineServiceBase_20230911085339.cs | 227 ++++++++++++ ...slationEngineServiceBase_20230911085341.cs | 227 ++++++++++++ ...slationEngineServiceBase_20230911085421.cs | 227 ++++++++++++ ...slationEngineServiceBase_20230911085505.cs | 227 ++++++++++++ ...slationEngineServiceBase_20230911085623.cs | 230 ++++++++++++ ...slationEngineServiceBase_20230911085631.cs | 227 ++++++++++++ ...slationEngineServiceBase_20230911085641.cs | 228 ++++++++++++ ...slationEngineServiceBase_20230911085646.cs | 228 ++++++++++++ ...slationEngineServiceBase_20230911085651.cs | 228 ++++++++++++ ...slationEngineServiceBase_20230911085722.cs | 228 ++++++++++++ ...slationEngineServiceBase_20230911085823.cs | 228 ++++++++++++ .../Usings_20230907084316.cs | 47 +++ .../Usings_20230907131743.cs | 47 +++ .../Usings_20230907141855.cs | 49 +++ .../Usings_20230907141856.cs | 48 +++ .../Usings_20230907141900.cs | 48 +++ .../Usings_20230907141901.cs | 48 +++ ...rMLNmtEngineServiceTests_20230906141540.cs | 192 ++++++++++ ...rMLNmtEngineServiceTests_20230907185104.cs | 192 ++++++++++ 220 files changed, 35526 insertions(+) create mode 100644 .history/.gitignore_20230911115750 create mode 100644 .history/.gitignore_20230911115828 create mode 100644 .history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs create mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs create mode 100644 .history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs create mode 100644 .history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs diff --git a/.history/.gitignore_20230911115750 b/.history/.gitignore_20230911115750 new file mode 100644 index 000000000..e1f430ccb --- /dev/null +++ b/.history/.gitignore_20230911115750 @@ -0,0 +1,51 @@ + +#ignore thumbnails created by windows +Thumbs.db +#Ignore files build by Visual Studio +*.obj +*.exe +*.pdb +*.user +*.aps +*.pch +*.vspscc +*_i.c +*_p.c +*.ncb +*.suo +*.tlb +*.tlh +*.bak +*.cache +*.ilk +*.log +[Bb]in +[Dd]ebug*/ +*.lib +*.sbr +obj/ +[Rr]elease*/ +_ReSharper*/ +[Tt]est[Rr]esult* + +*.Resharper +*.DotSettings +*- Copy* +packages/* +!packages/repositories.config +lib/ +*.files +*.VC.db +*.VC.opendb +*.lock.json +.vs +appsettings.user.json +artifacts +!**/Tes/release/ +thot-new-model.zip +*.tgz +out/ +src/sentencepiece4c/build/ +src/sentencepiece4cbuild/ +!samples/**/release +**/*.feature.cs diff --git a/.history/.gitignore_20230911115828 b/.history/.gitignore_20230911115828 new file mode 100644 index 000000000..fd44de4b5 --- /dev/null +++ b/.history/.gitignore_20230911115828 @@ -0,0 +1,52 @@ + +#ignore thumbnails created by windows +Thumbs.db +#Ignore files build by Visual Studio +*.obj +*.exe +*.pdb +*.user +*.aps +*.pch +*.vspscc +*_i.c +*_p.c +*.ncb +*.suo +*.tlb +*.tlh +*.bak +*.cache +*.ilk +*.log +[Bb]in +[Dd]ebug*/ +*.lib +*.sbr +obj/ +[Rr]elease*/ +_ReSharper*/ +[Tt]est[Rr]esult* +.history* + +*.Resharper +*.DotSettings +*- Copy* +packages/* +!packages/repositories.config +lib/ +*.files +*.VC.db +*.VC.opendb +*.lock.json +.vs +appsettings.user.json +artifacts +!**/Tes/release/ +thot-new-model.zip +*.tgz +out/ +src/sentencepiece4c/build/ +src/sentencepiece4cbuild/ +!samples/**/release +**/*.feature.cs diff --git a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs b/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs new file mode 100644 index 000000000..a8c7f1fd4 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs @@ -0,0 +1,317 @@ +using Microsoft.AspNetCore.Http; +using Serval.Translation.V1; + +namespace Microsoft.Extensions.DependencyInjection; + +public static class IMachineBuilderExtensions +{ + public static IMachineBuilder AddServiceOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddServiceOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddSmtTransferEngineOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddSmtTransferEngineOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddClearMLNmtEngineOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddClearMLNmtEngineOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddSharedFileOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddSharedFileOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddThotSmtModel( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder.AddThotSmtModel(); + } + + public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder.AddThotSmtModel(); + } + + public static IMachineBuilder AddTransferEngine(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddUnigramTruecaser(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddClearMLService(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); + + // workaround register satisfying the interface and as a hosted service. + builder.Services.AddSingleton(); + builder.Services.AddHostedService(p => p.GetRequiredService()); + + builder.Services + .AddHttpClient() + .AddTransientHttpErrorPolicy( + b => b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) + ); + + return builder; + } + + public static IMachineBuilder AddMongoBackgroundJobClient( + this IMachineBuilder builder, + string? connectionString = null + ) + { + builder.Services.AddHangfire( + c => + c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) + .UseSimpleAssemblyNameTypeSerializer() + .UseRecommendedSerializerSettings() + .UseMongoStorage( + connectionString ?? builder.Configuration.GetConnectionString("Hangfire"), + new MongoStorageOptions + { + MigrationOptions = new MongoMigrationOptions + { + MigrationStrategy = new MigrateMongoMigrationStrategy(), + BackupStrategy = new CollectionMongoBackupStrategy() + }, + CheckConnection = true, + CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, + } + ) + ); + builder.Services.AddHealthChecks().AddCheck(name: "Hangfire"); + return builder; + } + + public static IMachineBuilder AddBackgroundJobServer( + this IMachineBuilder builder, + IEnumerable? engineTypes = null + ) + { + engineTypes ??= + builder.Configuration?.GetSection("TranslationEngines").Get() + ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; + var queues = new List(); + foreach (TranslationEngineType engineType in engineTypes.Distinct()) + { + switch (engineType) + { + case TranslationEngineType.SmtTransfer: + builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); + queues.Add("smt_transfer"); + break; + case TranslationEngineType.Nmt: + builder.AddClearMLService(); + queues.Add("nmt"); + break; + } + } + + builder.Services.AddHangfireServer(o => + { + o.Queues = queues.ToArray(); + }); + return builder; + } + + public static IMachineBuilder AddMemoryDataAccess(this IMachineBuilder builder) + { + builder.Services.AddMemoryDataAccess(o => + { + o.AddRepository(); + o.AddRepository(); + o.AddRepository(); + }); + + return builder; + } + + public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, string? connectionString = null) + { + connectionString ??= builder.Configuration.GetConnectionString("Mongo"); + builder.Services.AddMongoDataAccess( + connectionString, + "SIL.Machine.AspNetCore.Models", + o => + { + o.AddRepository( + "translation_engines", + init: c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(p => p.EngineId) + ) + ) + ); + o.AddRepository( + "locks", + init: async c => + { + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending("writerLock._id")) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending("readerLocks._id")) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending("writerQueue._id")) + ); + } + ); + o.AddRepository( + "train_segment_pairs", + init: c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) + ) + ) + ); + } + ); + builder.Services.AddHealthChecks().AddMongoDb(connectionString, name: "Mongo"); + + return builder; + } + + public static IMachineBuilder AddServalPlatformService( + this IMachineBuilder builder, + string? connectionString = null + ) + { + builder.Services.AddScoped(); + builder.Services + .AddGrpcClient(o => + { + o.Address = new Uri(connectionString ?? builder.Configuration.GetConnectionString("Serval")); + }) + .ConfigureChannel(o => + { + o.MaxRetryAttempts = null; + o.ServiceConfig = new ServiceConfig + { + MethodConfigs = + { + new MethodConfig + { + Names = { MethodName.Default }, + RetryPolicy = new RetryPolicy + { + MaxAttempts = 10, + InitialBackoff = TimeSpan.FromSeconds(1), + MaxBackoff = TimeSpan.FromSeconds(5), + BackoffMultiplier = 1.5, + RetryableStatusCodes = { StatusCode.Unavailable } + } + }, + new MethodConfig + { + Names = + { + new MethodName + { + Service = "serval.translation.v1.TranslationPlatformApi", + Method = "UpdateBuildStatus" + } + } + }, + } + }; + }); + + return builder; + } + + public static IMachineBuilder AddServalTranslationEngineService( + this IMachineBuilder builder, + string? connectionString = null, + IEnumerable? engineTypes = null + ) + { + builder.Services.AddGrpc(options => options.Interceptors.Add()); + builder.AddServalPlatformService(connectionString ?? builder.Configuration.GetConnectionString("Serval")); + engineTypes ??= + builder.Configuration?.GetSection("TranslationEngines").Get() + ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; + foreach (TranslationEngineType engineType in engineTypes.Distinct()) + { + switch (engineType) + { + case TranslationEngineType.SmtTransfer: + builder.Services.AddSingleton(); + builder.Services.AddHostedService(); + builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); + builder.Services.AddScoped(); + break; + case TranslationEngineType.Nmt: + builder.AddClearMLService(); + builder.Services.AddScoped(); + break; + } + } + builder.Services.AddGrpcHealthChecks(); + + return builder; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs b/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs new file mode 100644 index 000000000..739a72924 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs @@ -0,0 +1,317 @@ +using Microsoft.AspNetCore.Http; +using Serval.Translation.V1; + +namespace Microsoft.Extensions.DependencyInjection; + +public static class IMachineBuilderExtensions +{ + public static IMachineBuilder AddServiceOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddServiceOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddSmtTransferEngineOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddSmtTransferEngineOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddClearMLNmtEngineOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddClearMLNmtEngineOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddSharedFileOptions( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder; + } + + public static IMachineBuilder AddSharedFileOptions(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder; + } + + public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddThotSmtModel( + this IMachineBuilder builder, + Action configureOptions + ) + { + builder.Services.Configure(configureOptions); + return builder.AddThotSmtModel(); + } + + public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder, IConfiguration config) + { + builder.Services.Configure(config); + return builder.AddThotSmtModel(); + } + + public static IMachineBuilder AddTransferEngine(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddUnigramTruecaser(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + return builder; + } + + public static IMachineBuilder AddClearMLService(this IMachineBuilder builder) + { + builder.Services.AddSingleton(); + builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); + + // workaround register satisfying the interface and as a hosted service. + builder.Services.AddSingleton(); + builder.Services.AddHostedService(p => p.GetRequiredService()); + + builder.Services + .AddHttpClient() + .AddTransientHttpErrorPolicy( + b => b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) + ); + + return builder; + } + + public static IMachineBuilder AddMongoBackgroundJobClient( + this IMachineBuilder builder, + string? connectionString = null + ) + { + builder.Services.AddHangfire( + c => + c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) + .UseSimpleAssemblyNameTypeSerializer() + .UseRecommendedSerializerSettings() + .UseMongoStorage( + connectionString ?? builder.Configuration.GetConnectionString("Hangfire"), + new MongoStorageOptions + { + MigrationOptions = new MongoMigrationOptions + { + MigrationStrategy = new MigrateMongoMigrationStrategy(), + BackupStrategy = new CollectionMongoBackupStrategy() + }, + CheckConnection = true, + CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, + } + ) + ); + builder.Services.AddHealthChecks().AddCheck(name: "Hangfire"); + return builder; + } + + public static IMachineBuilder AddBackgroundJobServer( + this IMachineBuilder builder, + IEnumerable? engineTypes = null + ) + { + engineTypes ??= + builder.Configuration?.GetSection("TranslationEngines").Get() + ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; + var queues = new List(); + foreach (TranslationEngineType engineType in engineTypes.Distinct()) + { + switch (engineType) + { + case TranslationEngineType.SmtTransfer: + builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); + queues.Add("smt_transfer"); + break; + case TranslationEngineType.Nmt: + builder.AddClearMLService(); + queues.Add("nmt"); + break; + } + } + + builder.Services.AddHangfireServer(o => + { + o.Queues = queues.ToArray(); + }); + return builder; + } + + public static IMachineBuilder AddMemoryDataAccess(this IMachineBuilder builder) + { + builder.Services.AddMemoryDataAccess(o => + { + o.AddRepository(); + o.AddRepository(); + o.AddRepository(); + }); + + return builder; + } + + public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, string? connectionString = null) + { + connectionString ??= builder.Configuration.GetConnectionString("Mongo"); + builder.Services.AddMongoDataAccess( + connectionString, + "SIL.Machine.AspNetCore.Models", + o => + { + o.AddRepository( + "translation_engines", + init: c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(p => p.EngineId) + ) + ) + ); + o.AddRepository( + "locks", + init: async c => + { + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending("writerLock._id")) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending("readerLocks._id")) + ); + await c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel(Builders.IndexKeys.Ascending("writerQueue._id")) + ); + } + ); + o.AddRepository( + "train_segment_pairs", + init: c => + c.Indexes.CreateOrUpdateAsync( + new CreateIndexModel( + Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) + ) + ) + ); + } + ); + builder.Services.AddHealthChecks().AddMongoDb(connectionString, name: "Mongo"); + + return builder; + } + + public static IMachineBuilder AddServalPlatformService( + this IMachineBuilder builder, + string? connectionString = null + ) + { + builder.Services.AddScoped(); + builder.Services + .AddGrpcClient(o => + { + o.Address = new Uri(connectionString ?? builder.Configuration.GetConnectionString("Serval")); + }) + .ConfigureChannel(o => + { + o.MaxRetryAttempts = null; + o.ServiceConfig = new ServiceConfig + { + MethodConfigs = + { + new MethodConfig + { + Names = { MethodName.Default }, + RetryPolicy = new Grpc.Net.Client.Configuration.RetryPolicy + { + MaxAttempts = 10, + InitialBackoff = TimeSpan.FromSeconds(1), + MaxBackoff = TimeSpan.FromSeconds(5), + BackoffMultiplier = 1.5, + RetryableStatusCodes = { StatusCode.Unavailable } + } + }, + new MethodConfig + { + Names = + { + new MethodName + { + Service = "serval.translation.v1.TranslationPlatformApi", + Method = "UpdateBuildStatus" + } + } + }, + } + }; + }); + + return builder; + } + + public static IMachineBuilder AddServalTranslationEngineService( + this IMachineBuilder builder, + string? connectionString = null, + IEnumerable? engineTypes = null + ) + { + builder.Services.AddGrpc(options => options.Interceptors.Add()); + builder.AddServalPlatformService(connectionString ?? builder.Configuration.GetConnectionString("Serval")); + engineTypes ??= + builder.Configuration?.GetSection("TranslationEngines").Get() + ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; + foreach (TranslationEngineType engineType in engineTypes.Distinct()) + { + switch (engineType) + { + case TranslationEngineType.SmtTransfer: + builder.Services.AddSingleton(); + builder.Services.AddHostedService(); + builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); + builder.Services.AddScoped(); + break; + case TranslationEngineType.Nmt: + builder.AddClearMLService(); + builder.Services.AddScoped(); + break; + } + } + builder.Services.AddGrpcHealthChecks(); + + return builder; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs new file mode 100644 index 000000000..918f40b90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs new file mode 100644 index 000000000..918f40b90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs new file mode 100644 index 000000000..9e7eed761 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs @@ -0,0 +1,343 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + await sourceTrainWriter.BaseStream.DisposeAsync(); + await sourceTrainWriter.BaseStream.DisposeAsync(); + + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs new file mode 100644 index 000000000..e41df89d3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs @@ -0,0 +1,342 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + await sourceTrainWriter.BaseStream.DisposeAsync(); + await targetTrainWriter.BaseStream.DisposeAsync(); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs new file mode 100644 index 000000000..e41df89d3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs @@ -0,0 +1,342 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + await sourceTrainWriter.BaseStream.DisposeAsync(); + await targetTrainWriter.BaseStream.DisposeAsync(); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs new file mode 100644 index 000000000..e41df89d3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs @@ -0,0 +1,342 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + await sourceTrainWriter.BaseStream.DisposeAsync(); + await targetTrainWriter.BaseStream.DisposeAsync(); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs new file mode 100644 index 000000000..e41df89d3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs @@ -0,0 +1,342 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + await sourceTrainWriter.BaseStream.DisposeAsync(); + await targetTrainWriter.BaseStream.DisposeAsync(); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs new file mode 100644 index 000000000..bf3981d14 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs @@ -0,0 +1,342 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + await sourceTrainWriter.BaseStream.DisposeAsync(); + await targetTrainWriter.BaseStream.DisposeAsync(); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs new file mode 100644 index 000000000..a26cda901 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs new file mode 100644 index 000000000..918f40b90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs new file mode 100644 index 000000000..918f40b90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs new file mode 100644 index 000000000..918f40b90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs new file mode 100644 index 000000000..918f40b90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs new file mode 100644 index 000000000..918f40b90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs @@ -0,0 +1,340 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineBuildJob +{ + private readonly IPlatformService _platformService; + private readonly IRepository _engines; + private readonly ILogger _logger; + private readonly IClearMLService _clearMLService; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + private readonly ICorpusService _corpusService; + + public ClearMLNmtEngineBuildJob( + IPlatformService platformService, + IRepository engines, + ILogger logger, + IClearMLService clearMLService, + ISharedFileService sharedFileService, + IOptionsMonitor options, + ICorpusService corpusService + ) + { + _platformService = platformService; + _engines = engines; + _logger = logger; + _clearMLService = clearMLService; + _sharedFileService = sharedFileService; + _options = options; + _corpusService = corpusService; + } + + [Queue("nmt")] + [AutomaticRetry(Attempts = 0)] + public async Task RunAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); + if (clearMLProjectId is null) + return; + + try + { + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken: cancellationToken + ); + if (engine is null || engine.IsCanceled) + throw new OperationCanceledException(); + + int corpusSize; + if (engine.BuildState is BuildState.Pending) + corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); + else + corpusSize = GetCorpusSize(corpora); + + string clearMLTaskId; + ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); + if (clearMLTask is null) + { + clearMLTaskId = await _clearMLService.CreateTaskAsync( + buildId, + clearMLProjectId, + engineId, + engine.SourceLanguage, + engine.TargetLanguage, + _sharedFileService.GetBaseUri().ToString(), + cancellationToken + ); + await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); + } + else + { + clearMLTaskId = clearMLTask.Id; + } + + int lastIteration = 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); + if (clearMLTask is null) + throw new InvalidOperationException("The ClearML task does not exist."); + + if ( + engine.BuildState == BuildState.Pending + && clearMLTask.Status + is ClearMLTaskStatus.InProgress + or ClearMLTaskStatus.Stopped + or ClearMLTaskStatus.Failed + or ClearMLTaskStatus.Completed + ) + { + engine = await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, + u => u.Set(e => e.BuildState, BuildState.Active), + cancellationToken: cancellationToken + ); + if (engine is null) + throw new OperationCanceledException(); + await _platformService.BuildStartedAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build started ({0})", buildId); + } + + switch (clearMLTask.Status) + { + case ClearMLTaskStatus.InProgress: + case ClearMLTaskStatus.Completed: + if (lastIteration != clearMLTask.LastIteration) + { + await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); + lastIteration = clearMLTask.LastIteration; + } + break; + case ClearMLTaskStatus.Stopped: + // This could have been triggered from the ClearML UI, so set IsCanceled to true. + await _engines.UpdateAsync( + e => e.EngineId == engineId && !e.IsCanceled, + u => u.Set(e => e.IsCanceled, true), + cancellationToken: CancellationToken.None + ); + throw new OperationCanceledException(); + case ClearMLTaskStatus.Failed: + throw new InvalidOperationException(clearMLTask.StatusReason); + } + if (clearMLTask.Status is ClearMLTaskStatus.Completed) + break; + await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); + } + + // The ClearML task has successfully completed, so insert the generated pretranslations into the database. + await InsertPretranslationsAsync(engineId, buildId, cancellationToken); + + IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( + clearMLTaskId, + CancellationToken.None + ); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Inc(e => e.BuildRevision) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (!metrics.TryGetValue("bleu", out double confidence)) + confidence = 0; + + await _platformService.BuildCompletedAsync( + buildId, + corpusSize, + Math.Round(confidence, 2, MidpointRounding.AwayFromZero), + CancellationToken.None + ); + _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); + } + catch (OperationCanceledException) + { + // Check if the cancellation was initiated by an API call or a shutdown. + TranslationEngine? engine = await _engines.GetAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + CancellationToken.None + ); + if (engine is null || engine.IsCanceled) + { + // This is an actual cancellation triggered by an API call. + ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); + if (task is not null) + await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); + + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + + bool buildStarted = await _engines.ExistsAsync( + e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, + CancellationToken.None + ); + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + if (buildStarted) + { + await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); + _logger.LogInformation("Build canceled ({0})", buildId); + } + } + else if (engine is not null) + { + // the build was canceled, because of a server shutdown + // switch state back to pending + await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); + } + + throw; + } + catch (Exception e) + { + _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); + + try + { + await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); + } + catch (Exception e2) + { + _logger.LogError( + $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." + ); + } + + await _engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + u => + u.Set(e => e.BuildState, BuildState.None) + .Set(e => e.IsCanceled, false) + .Unset(e => e.JobId) + .Unset(e => e.BuildId), + cancellationToken: CancellationToken.None + ); + + await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); + throw; + } + } + + private async Task WriteDataFilesAsync( + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + await using var sourceTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) + ); + await using var targetTrainWriter = new StreamWriter( + await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) + ); + + int corpusSize = 0; + async IAsyncEnumerable ProcessRowsAsync() + { + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( + targetCorpus, + allSourceRows: true, + allTargetRows: true + ); + + foreach (ParallelTextRow row in parallelCorpus) + { + await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); + await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); + if ( + (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) + && row.SourceSegment.Count > 0 + && row.TargetSegment.Count == 0 + ) + { + yield return new Pretranslation + { + CorpusId = corpus.Id, + TextId = row.TextId, + Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), + Translation = row.SourceText + }; + } + if (!row.IsEmpty) + corpusSize++; + } + } + } + + await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( + $"builds/{buildId}/pretranslate.src.json", + cancellationToken + ); + + await JsonSerializer.SerializeAsync( + sourcePretranslateStream, + ProcessRowsAsync(), + new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken: cancellationToken + ); + return corpusSize; + } + + private int GetCorpusSize(IReadOnlyList corpora) + { + int corpusSize = 0; + foreach (Corpus corpus in corpora) + { + ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); + ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); + + IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); + + corpusSize += parallelCorpus.Count(includeEmpty: false); + } + return corpusSize; + } + + private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) + { + await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( + $"builds/{buildId}/pretranslate.trg.json", + cancellationToken + ); + + IAsyncEnumerable pretranslations = JsonSerializer + .DeserializeAsyncEnumerable( + targetPretranslateStream, + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, + cancellationToken + ) + .OfType(); + + await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs new file mode 100644 index 000000000..42eb064e1 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs @@ -0,0 +1,51 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineService : TranslationEngineServiceBase +{ + private readonly IClearMLService _clearMLService; + + public ClearMLNmtEngineService( + IBackgroundJobClient jobClient, + IPlatformService platformService, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + IRepository engines, + IClearMLService clearMLService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _clearMLService = clearMLService; + } + + public override TranslationEngineType Type => TranslationEngineType.Nmt; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + await _clearMLService.CreateProjectAsync(engineId, engineName, cancellationToken: CancellationToken.None); + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + string? projectId = await _clearMLService.GetProjectIdAsync(engineId, CancellationToken.None); + if (projectId is not null) + await _clearMLService.DeleteProjectAsync(projectId, CancellationToken.None); + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs new file mode 100644 index 000000000..42eb064e1 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs @@ -0,0 +1,51 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineService : TranslationEngineServiceBase +{ + private readonly IClearMLService _clearMLService; + + public ClearMLNmtEngineService( + IBackgroundJobClient jobClient, + IPlatformService platformService, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + IRepository engines, + IClearMLService clearMLService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _clearMLService = clearMLService; + } + + public override TranslationEngineType Type => TranslationEngineType.Nmt; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + await _clearMLService.CreateProjectAsync(engineId, engineName, cancellationToken: CancellationToken.None); + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + string? projectId = await _clearMLService.GetProjectIdAsync(engineId, CancellationToken.None); + if (projectId is not null) + await _clearMLService.DeleteProjectAsync(projectId, CancellationToken.None); + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs new file mode 100644 index 000000000..42eb064e1 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs @@ -0,0 +1,51 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class ClearMLNmtEngineService : TranslationEngineServiceBase +{ + private readonly IClearMLService _clearMLService; + + public ClearMLNmtEngineService( + IBackgroundJobClient jobClient, + IPlatformService platformService, + IDistributedReaderWriterLockFactory lockFactory, + IDataAccessContext dataAccessContext, + IRepository engines, + IClearMLService clearMLService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _clearMLService = clearMLService; + } + + public override TranslationEngineType Type => TranslationEngineType.Nmt; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + await _clearMLService.CreateProjectAsync(engineId, engineName, cancellationToken: CancellationToken.None); + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + string? projectId = await _clearMLService.GetProjectIdAsync(engineId, CancellationToken.None); + if (projectId is not null) + await _clearMLService.DeleteProjectAsync(projectId, CancellationToken.None); + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs new file mode 100644 index 000000000..56bbc9829 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs @@ -0,0 +1,95 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + return Task.FromResult( + new BufferedStream(new S3WriteStream(_client, objectId, _bucketName), 1024 * 1024 * 100) + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs new file mode 100644 index 000000000..56bbc9829 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs @@ -0,0 +1,95 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + return Task.FromResult( + new BufferedStream(new S3WriteStream(_client, objectId, _bucketName), 1024 * 1024 * 100) + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs new file mode 100644 index 000000000..aaf4a3f52 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs @@ -0,0 +1,97 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + 1024 * 1024 * 5 + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs new file mode 100644 index 000000000..66204bcaa --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs @@ -0,0 +1,97 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + 1024 * 1024 * 5)); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs new file mode 100644 index 000000000..413860de7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs @@ -0,0 +1,98 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + 1024 * 1024 * 5 + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs new file mode 100644 index 000000000..413860de7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs @@ -0,0 +1,98 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + 1024 * 1024 * 5 + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs new file mode 100644 index 000000000..1eeacbc76 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs @@ -0,0 +1,98 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + S3WriteStream.FiveMB + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs new file mode 100644 index 000000000..1eeacbc76 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs @@ -0,0 +1,98 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + + public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + S3WriteStream.FiveMB + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs new file mode 100644 index 000000000..2ae672c3c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs @@ -0,0 +1,107 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + private readonly ILoggerFactory _loggerFactory; + + public S3FileStorage( + string bucketName, + string basePath, + string accessKeyId, + string secretAccessKey, + string region, + ILoggerFactory loggerFactory + ) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = Amazon.Runtime.RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + _loggerFactory = loggerFactory; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + S3WriteStream.FiveMB + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs new file mode 100644 index 000000000..0619ee785 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs @@ -0,0 +1,107 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + private readonly ILoggerFactory _loggerFactory; + + public S3FileStorage( + string bucketName, + string basePath, + string accessKeyId, + string secretAccessKey, + string region, + ILoggerFactory loggerFactory + ) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + _loggerFactory = loggerFactory; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + S3WriteStream.FiveMB + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs new file mode 100644 index 000000000..b5eb883cb --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs @@ -0,0 +1,108 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + private readonly ILoggerFactory _loggerFactory; + + public S3FileStorage( + string bucketName, + string basePath, + string accessKeyId, + string secretAccessKey, + string region, + ILoggerFactory loggerFactory + ) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + _loggerFactory = loggerFactory; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId), + S3WriteStream.FiveMB, + _loggerFactory + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs new file mode 100644 index 000000000..9058da211 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs @@ -0,0 +1,107 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + private readonly ILoggerFactory _loggerFactory; + + public S3FileStorage( + string bucketName, + string basePath, + string accessKeyId, + string secretAccessKey, + string region, + ILoggerFactory loggerFactory + ) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + _loggerFactory = loggerFactory; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId, _loggerFactory), + S3WriteStream.FiveMB + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs new file mode 100644 index 000000000..9058da211 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs @@ -0,0 +1,107 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + private readonly ILoggerFactory _loggerFactory; + + public S3FileStorage( + string bucketName, + string basePath, + string accessKeyId, + string secretAccessKey, + string region, + ILoggerFactory loggerFactory + ) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + _loggerFactory = loggerFactory; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId, _loggerFactory), + S3WriteStream.FiveMB + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs new file mode 100644 index 000000000..ff8563a42 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs @@ -0,0 +1,107 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3FileStorage : FileStorage +{ + private readonly AmazonS3Client _client; + private readonly string _bucketName; + private readonly string _basePath; + private readonly ILoggerFactory _loggerFactory; + + public S3FileStorage( + string bucketName, + string basePath, + string accessKeyId, + string secretAccessKey, + string region, + ILoggerFactory loggerFactory + ) + { + _client = new AmazonS3Client( + accessKeyId, + secretAccessKey, + new AmazonS3Config + { + RetryMode = RequestRetryMode.Standard, + MaxErrorRetry = 3, + RegionEndpoint = RegionEndpoint.GetBySystemName(region) + } + ); + + _bucketName = bucketName; + //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation + _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; + _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; + _loggerFactory = loggerFactory; + } + + public override void Dispose() { } + + public override async Task Exists(string path, CancellationToken cancellationToken = default) + { + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), + MaxKeys = 1 + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + + return response.S3Objects.Any(); + } + + public override async Task> Ls( + string? path = null, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + if (path != null && !path.EndsWith("/")) + throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); + + var request = new ListObjectsV2Request + { + BucketName = _bucketName, + Prefix = _basePath + Normalize(path, includeTrailingSlash: true), + MaxKeys = 1, + Delimiter = recurse ? "" : "/" + }; + + ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); + return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); + } + + public override async Task OpenRead(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new FileNotFoundException($"File {objectId} does not exist"); + return response.ResponseStream; + } + + public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) + { + string objectId = _basePath + Normalize(path); + InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; + InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); + return new BufferedStream( + new S3WriteStream(_client, objectId, _bucketName, response.UploadId, _loggerFactory), + S3WriteStream.MaxPartSize + ); + } + + public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) + { + if (path is null) + throw new ArgumentNullException(nameof(path)); + string objectId = _basePath + Normalize(path); + DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; + DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); + if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) + new HttpRequestException( + $"Received status code {response.HttpStatusCode} when attempting to delete {path}" + ); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs new file mode 100644 index 000000000..b2d5808f0 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs @@ -0,0 +1,76 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _bucketName; + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName) + { + _client = client; + _key = key; + _bucketName = bucketName; + _length = 0; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + await transferUtility.UploadAsync(uploadRequest); + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs new file mode 100644 index 000000000..e27f3416a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs @@ -0,0 +1,76 @@ +namespace SIL.Machine.AspNetCore.Services;S3Wri + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _bucketName; + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName) + { + _client = client; + _key = key; + _bucketName = bucketName; + _length = 0; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + await transferUtility.UploadAsync(uploadRequest); + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs new file mode 100644 index 000000000..b2d5808f0 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs @@ -0,0 +1,76 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _bucketName; + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName) + { + _client = client; + _key = key; + _bucketName = bucketName; + _length = 0; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + await transferUtility.UploadAsync(uploadRequest); + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs new file mode 100644 index 000000000..a4284a6fe --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs @@ -0,0 +1,77 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _bucketName; + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName) + { + _client = client; + _key = key; + _bucketName = bucketName; + _length = 0; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + await transferUtility.UploadAsync(uploadRequest); + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs new file mode 100644 index 000000000..effbfca90 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs @@ -0,0 +1,79 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + await transferUtility.UploadAsync(uploadRequest); + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs new file mode 100644 index 000000000..1fcc576e6 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs @@ -0,0 +1,78 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + await transferUtility.UploadAsync(uploadRequest); + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs new file mode 100644 index 000000000..213ddd80a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs @@ -0,0 +1,78 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + await transferUtility.UploadAsync(uploadRequest); + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs new file mode 100644 index 000000000..24f757833 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs @@ -0,0 +1,87 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) + { + using Stream inputStream = new MemoryStream(buffer, offset, count); + using var transferUtility = new TransferUtility(_client); + var uploadRequest = new TransferUtilityUploadRequest + { + BucketName = _bucketName, + InputStream = inputStream, + Key = _key, + PartSize = count + }; + transferUtility.Upload(uploadRequest); + } + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + FilePosition = _length + }; + _length += count; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs new file mode 100644 index 000000000..04454309b --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs @@ -0,0 +1,75 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + FilePosition = _length + }; + _length += count; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + public override ValueTask DisposeAsync() + { + Dispose(disposing: false); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs new file mode 100644 index 000000000..46a493f7f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs @@ -0,0 +1,108 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + FilePosition = _length + }; + _length += count; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs new file mode 100644 index 000000000..46a493f7f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs @@ -0,0 +1,108 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + FilePosition = _length + }; + _length += count; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs new file mode 100644 index 000000000..a5cdde5c5 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs @@ -0,0 +1,106 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber + }; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs new file mode 100644 index 000000000..16f137a5a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs @@ -0,0 +1,107 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber + }; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs new file mode 100644 index 000000000..7d72d7d4d --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs @@ -0,0 +1,108 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + Stream = ms + }; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs new file mode 100644 index 000000000..21f06731c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs @@ -0,0 +1,108 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs new file mode 100644 index 000000000..21f06731c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs @@ -0,0 +1,108 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + uploadResponses.Add(await _client.UploadPartAsync(request)); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs new file mode 100644 index 000000000..c551af600 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs @@ -0,0 +1,109 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + UploadPartResponse response = await _client.UploadPartAsync(request); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs new file mode 100644 index 000000000..859618376 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs @@ -0,0 +1,110 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + UploadPartResponse response = await _client.UploadPartAsync(request); + if(response.HttpStatusCode != HttpStatusCode.OK) throw new HttpRequestException($"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}") + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs new file mode 100644 index 000000000..38f8cf6d7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs @@ -0,0 +1,113 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs new file mode 100644 index 000000000..4e5370496 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs @@ -0,0 +1,116 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => _length; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + uploadRequest.StreamTransferProgress += new EventHandler( + UploadPartProgressEventCallback + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs new file mode 100644 index 000000000..e89e96826 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs @@ -0,0 +1,114 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private long _length; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler((_, e) => { }); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs new file mode 100644 index 000000000..7be03bf75 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs @@ -0,0 +1,113 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler((_, e) => { }); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs new file mode 100644 index 000000000..785cf5858 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs @@ -0,0 +1,114 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler((_, e) => { }); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs new file mode 100644 index 000000000..64918822a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs @@ -0,0 +1,115 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = (new LoggerFactory()).CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler((_, e) => { }); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs new file mode 100644 index 000000000..3a05d9ab0 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs @@ -0,0 +1,115 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler((_, e) => { }); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs new file mode 100644 index 000000000..2d1c7f47a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs @@ -0,0 +1,120 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs new file mode 100644 index 000000000..2d1c7f47a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs @@ -0,0 +1,120 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception) + { + await Abort(); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort() + { + // Logging? + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs new file mode 100644 index 000000000..7c5630f4b --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs @@ -0,0 +1,120 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs new file mode 100644 index 000000000..7c5630f4b --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs @@ -0,0 +1,120 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception) + { + await Abort(); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs new file mode 100644 index 000000000..52ecc2ef3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs @@ -0,0 +1,120 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs new file mode 100644 index 000000000..52ecc2ef3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs @@ -0,0 +1,120 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs new file mode 100644 index 000000000..cb6f6edb5 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs @@ -0,0 +1,120 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs new file mode 100644 index 000000000..d6bc7c0aa --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs @@ -0,0 +1,125 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs new file mode 100644 index 000000000..81005c673 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs @@ -0,0 +1,125 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs new file mode 100644 index 000000000..bff5aa1d7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs @@ -0,0 +1,125 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs new file mode 100644 index 000000000..ac7aa225c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs @@ -0,0 +1,124 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs new file mode 100644 index 000000000..5f5b59245 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs @@ -0,0 +1,125 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs new file mode 100644 index 000000000..530e928ca --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs @@ -0,0 +1,124 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs new file mode 100644 index 000000000..530e928ca --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs @@ -0,0 +1,124 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List uploadResponses = new(); + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs new file mode 100644 index 000000000..7e13f6a62 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs @@ -0,0 +1,125 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs new file mode 100644 index 000000000..104dc7116 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs @@ -0,0 +1,125 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs new file mode 100644 index 000000000..104dc7116 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs @@ -0,0 +1,125 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs new file mode 100644 index 000000000..a97937131 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs @@ -0,0 +1,126 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs new file mode 100644 index 000000000..a97937131 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs @@ -0,0 +1,126 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs new file mode 100644 index 000000000..a97937131 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs @@ -0,0 +1,126 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs new file mode 100644 index 000000000..1024e4aec --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs @@ -0,0 +1,127 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) { } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs new file mode 100644 index 000000000..4ef04e880 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs @@ -0,0 +1,151 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs new file mode 100644 index 000000000..e4c5eb276 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs @@ -0,0 +1,151 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client.CompleteMultipartUpload(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs new file mode 100644 index 000000000..9a03bb2e4 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs new file mode 100644 index 000000000..df9aba354 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs @@ -0,0 +1,155 @@ +using Nito.AsyncEx.Synchronous; + +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs new file mode 100644 index 000000000..584a0e09c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs @@ -0,0 +1,153 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs new file mode 100644 index 000000000..9b4f9cf96 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public static final int fiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs new file mode 100644 index 000000000..d7b0b6982 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public final static int fiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs new file mode 100644 index 000000000..a47284fb5 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public static const int fiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs new file mode 100644 index 000000000..50a58fafe --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int fiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs new file mode 100644 index 000000000..c13ee9bc2 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FIVE_MB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs new file mode 100644 index 000000000..c13ee9bc2 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FIVE_MB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs new file mode 100644 index 000000000..20cee4645 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMb = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs new file mode 100644 index 000000000..262f9a6c3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMbB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs new file mode 100644 index 000000000..9ea77dfee --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs new file mode 100644 index 000000000..9ea77dfee --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = 5 * 1024 * 1024 + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs new file mode 100644 index 000000000..6d25881bb --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs new file mode 100644 index 000000000..6d25881bb --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs @@ -0,0 +1,155 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs new file mode 100644 index 000000000..95dd7899d --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs @@ -0,0 +1,161 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = new LoggerFactory().CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs new file mode 100644 index 000000000..001301acb --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs @@ -0,0 +1,161 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs new file mode 100644 index 000000000..001301acb --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs @@ -0,0 +1,161 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs new file mode 100644 index 000000000..7fc4c6f7a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs @@ -0,0 +1,162 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs new file mode 100644 index 000000000..afd96e3f9 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs @@ -0,0 +1,163 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + throw; + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs new file mode 100644 index 000000000..afd96e3f9 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs @@ -0,0 +1,163 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await Abort(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + Abort(e).WaitAndUnwrapException(); + throw; + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await Abort(e); + } + } + + private async Task Abort(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs new file mode 100644 index 000000000..339ca3a26 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs @@ -0,0 +1,163 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await AbortAsync(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + AbortAsync(e).WaitAndUnwrapException(); + throw; + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await AbortAsync(e); + } + } + + private async Task AbortAsync(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs new file mode 100644 index 000000000..339ca3a26 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs @@ -0,0 +1,163 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int FiveMB = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = FiveMB + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await AbortAsync(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + AbortAsync(e).WaitAndUnwrapException(); + throw; + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await AbortAsync(e); + } + } + + private async Task AbortAsync(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs new file mode 100644 index 000000000..fc2173053 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs @@ -0,0 +1,163 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int MaxPartSize = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = MaxPartSize + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await AbortAsync(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + AbortAsync(e).WaitAndUnwrapException(); + throw; + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await AbortAsync(e); + } + } + + private async Task AbortAsync(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs new file mode 100644 index 000000000..fc2173053 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs @@ -0,0 +1,163 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int MaxPartSize = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = MaxPartSize + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await AbortAsync(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + AbortAsync(e).WaitAndUnwrapException(); + throw; + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await AbortAsync(e); + } + } + + private async Task AbortAsync(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs new file mode 100644 index 000000000..fc2173053 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs @@ -0,0 +1,163 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class S3WriteStream : Stream +{ + private readonly AmazonS3Client _client; + private readonly string _key; + private readonly string _uploadId; + private readonly string _bucketName; + private readonly List _uploadResponses; + private readonly ILogger _logger; + + public const int MaxPartSize = 5 * 1024 * 1024; + + public S3WriteStream( + AmazonS3Client client, + string key, + string bucketName, + string uploadId, + ILoggerFactory loggerFactory + ) + { + _client = client; + _key = key; + _bucketName = bucketName; + _uploadId = uploadId; + _logger = loggerFactory.CreateLogger(); + _uploadResponses = new List(); + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => 0; + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() { } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + try + { + using MemoryStream ms = new(buffer, offset, count); + int partNumber = _uploadResponses.Count + 1; + UploadPartRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId, + PartNumber = partNumber, + InputStream = ms, + PartSize = MaxPartSize + }; + request.StreamTransferProgress += new EventHandler( + (_, e) => + { + _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); + } + ); + UploadPartResponse response = await _client.UploadPartAsync(request); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + _uploadResponses.Add(response); + } + catch (Exception e) + { + await AbortAsync(e); + throw; + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = _client + .CompleteMultipartUploadAsync(request) + .WaitAndUnwrapException(); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + AbortAsync(e).WaitAndUnwrapException(); + throw; + } + } + base.Dispose(disposing); + } + + public async override ValueTask DisposeAsync() + { + try + { + CompleteMultipartUploadRequest request = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + request.AddPartETags(_uploadResponses); + CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); + Dispose(disposing: false); + GC.SuppressFinalize(this); + if (response.HttpStatusCode != HttpStatusCode.OK) + throw new HttpRequestException( + $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" + ); + } + catch (Exception e) + { + await AbortAsync(e); + } + } + + private async Task AbortAsync(Exception e) + { + _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); + AbortMultipartUploadRequest abortMPURequest = + new() + { + BucketName = _bucketName, + Key = _key, + UploadId = _uploadId + }; + await _client.AbortMultipartUploadAsync(abortMPURequest); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs new file mode 100644 index 000000000..526cf95fd --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs @@ -0,0 +1,92 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + + public SharedFileService(IOptions? options = null) + { + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs new file mode 100644 index 000000000..526cf95fd --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs @@ -0,0 +1,92 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + + public SharedFileService(IOptions? options = null) + { + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs new file mode 100644 index 000000000..a7a778353 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs @@ -0,0 +1,92 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + + public SharedFileService(IOptions? options = null) + { + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + }} + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs new file mode 100644 index 000000000..526cf95fd --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs @@ -0,0 +1,92 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + + public SharedFileService(IOptions? options = null) + { + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs new file mode 100644 index 000000000..c7eb4f847 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs @@ -0,0 +1,95 @@ +using SIL.Reporting; + +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) + { + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs new file mode 100644 index 000000000..53777aeaa --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs @@ -0,0 +1,97 @@ +using SIL.Reporting; + +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) + { + _loggerFactory = loggerFactory; + + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs new file mode 100644 index 000000000..c9f4aa6b1 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs @@ -0,0 +1,98 @@ +using SIL.Reporting; + +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) + { + _loggerFactory = loggerFactory; + + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region, + _loggerFactory + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs new file mode 100644 index 000000000..c9f4aa6b1 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs @@ -0,0 +1,98 @@ +using SIL.Reporting; + +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) + { + _loggerFactory = loggerFactory; + + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region, + _loggerFactory + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs new file mode 100644 index 000000000..5804584c1 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs @@ -0,0 +1,98 @@ +using SIL.Reporting; + +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) + { + _loggerFactory = loggerFactory; + + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region, + _loggerFactory + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs new file mode 100644 index 000000000..5804584c1 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs @@ -0,0 +1,98 @@ +using SIL.Reporting; + +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) + { + _loggerFactory = loggerFactory; + + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region, + _loggerFactory + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs new file mode 100644 index 000000000..cef73bbeb --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs @@ -0,0 +1,96 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) + { + _loggerFactory = loggerFactory; + + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region, + _loggerFactory + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs new file mode 100644 index 000000000..cef73bbeb --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs @@ -0,0 +1,96 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SharedFileService : ISharedFileService +{ + private readonly Uri? _baseUri; + private readonly FileStorage _fileStorage; + private readonly bool _supportFolderDelete = true; + private readonly ILoggerFactory _loggerFactory; + + public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) + { + _loggerFactory = loggerFactory; + + if (options?.Value.Uri is null) + { + _fileStorage = new InMemoryStorage(); + } + else + { + string baseUri = options.Value.Uri; + if (!baseUri.EndsWith("/")) + baseUri += "/"; + _baseUri = new Uri(baseUri); + switch (_baseUri.Scheme) + { + case "file": + _fileStorage = new LocalStorage(_baseUri.LocalPath); + Directory.CreateDirectory(_baseUri.LocalPath); + break; + case "s3": + _fileStorage = new S3FileStorage( + _baseUri.Host, + _baseUri.AbsolutePath, + options.Value.S3AccessKeyId, + options.Value.S3SecretAccessKey, + options.Value.S3Region, + _loggerFactory + ); + _supportFolderDelete = false; + break; + default: + throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); + } + } + } + + public Uri GetBaseUri() + { + return GetResolvedUri(""); + } + + public Uri GetResolvedUri(string path) + { + if (_baseUri is null) + return new Uri($"memory://{path}"); + return new Uri(_baseUri, path); + } + + public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenRead(path, cancellationToken); + } + + public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.OpenWrite(path, cancellationToken); + } + + public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) + { + if (!_supportFolderDelete && path.EndsWith("/")) + { + IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); + foreach (string file in files) + await _fileStorage.Rm(file, cancellationToken: cancellationToken); + } + else + { + await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); + } + } + + public Task ExistsAsync(string path, CancellationToken cancellationToken = default) + { + return _fileStorage.Exists(path, cancellationToken); + } + + public Task> Ls( + string path, + bool recurse = false, + CancellationToken cancellationToken = default + ) + { + return _fileStorage.Ls(path, recurse, cancellationToken); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs new file mode 100644 index 000000000..e7619bc63 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs @@ -0,0 +1,181 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs new file mode 100644 index 000000000..804a21c2e --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs @@ -0,0 +1,183 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + if (state.CurrentBuildRevision == -1) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs new file mode 100644 index 000000000..804a21c2e --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs @@ -0,0 +1,183 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + if (state.CurrentBuildRevision == -1) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs new file mode 100644 index 000000000..804a21c2e --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs @@ -0,0 +1,183 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + if (state.CurrentBuildRevision == -1) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs new file mode 100644 index 000000000..68d6dc96e --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs @@ -0,0 +1,183 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + if (state.CurrentBuildRevision == -1) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs new file mode 100644 index 000000000..38b27e746 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if (await Engines.GetAsync(e => e.Id == engineId && e.BuildId)) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs new file mode 100644 index 000000000..4c6134aa9 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if ( + ( + await Engines.GetAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs new file mode 100644 index 000000000..cf92eef1a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if ( + ( + await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs new file mode 100644 index 000000000..cf92eef1a --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if ( + ( + await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs new file mode 100644 index 000000000..4672cf290 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if ( + !( + await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs new file mode 100644 index 000000000..4672cf290 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if ( + !( + await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs new file mode 100644 index 000000000..f75ec89f0 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs @@ -0,0 +1,190 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if ( + !( + await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs new file mode 100644 index 000000000..f75ec89f0 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs @@ -0,0 +1,190 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine e; + if ( + !( + await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs new file mode 100644 index 000000000..22727b2d7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs @@ -0,0 +1,189 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + if ( + !( + await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs new file mode 100644 index 000000000..22727b2d7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs @@ -0,0 +1,189 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + if ( + !( + await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs new file mode 100644 index 000000000..22727b2d7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs @@ -0,0 +1,189 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + if ( + !( + await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs new file mode 100644 index 000000000..22727b2d7 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs @@ -0,0 +1,189 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + if ( + !( + await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ) + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs new file mode 100644 index 000000000..f5d00996f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs @@ -0,0 +1,190 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + if ( + !( + + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs new file mode 100644 index 000000000..dd3aa1037 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs @@ -0,0 +1,192 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach(var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if ( + !( + + ).Any() + ) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs new file mode 100644 index 000000000..b773a87c2 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!(engines).Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs new file mode 100644 index 000000000..ec235055c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs new file mode 100644 index 000000000..46ad52609 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs @@ -0,0 +1,189 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + Console.WriteLine("|||||||||||||||||||"); + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs new file mode 100644 index 000000000..9991f0c87 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs @@ -0,0 +1,191 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + Console.WriteLine("|||||||||||||||||||"); + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + Console.WriteLine("|||||||||||||||||||"); + + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs new file mode 100644 index 000000000..3574fa79f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs @@ -0,0 +1,190 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + Console.WriteLine("|||||||||||||||||||"); + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + Console.WriteLine("|||||||||||||||||||"); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs new file mode 100644 index 000000000..47ce83b1f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs @@ -0,0 +1,190 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + Console.WriteLine("|||||||||||||||||||"); + IEnumerable engines = await Engines.GetAllAsync( + e => e.Id == engineId && e.BuildState == BuildState.None + ); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + Console.WriteLine("|||||||||||||||||||"); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs new file mode 100644 index 000000000..234a20433 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs @@ -0,0 +1,188 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + Console.WriteLine("|||||||||||||||||||"); + IEnumerable engines = await Engines.GetAllAsync(e => e.Id == engineId); + foreach (var eng in engines) + Console.WriteLine(JsonSerializer.Serialize(eng)); + Console.WriteLine("|||||||||||||||||||"); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs new file mode 100644 index 000000000..92d3d3d1e --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs @@ -0,0 +1,185 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs new file mode 100644 index 000000000..26186680c --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs new file mode 100644 index 000000000..eaa649a10 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs @@ -0,0 +1,185 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (!engines.Any()) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs new file mode 100644 index 000000000..76069459f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildRevision == 0 || engine.BuildState != BuildState.None || engine.Revision == 0) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs new file mode 100644 index 000000000..76069459f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildRevision == 0 || engine.BuildState != BuildState.None || engine.Revision == 0) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs new file mode 100644 index 000000000..76069459f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildRevision == 0 || engine.BuildState != BuildState.None || engine.Revision == 0) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs new file mode 100644 index 000000000..b73ba6ead --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.Revision == 0) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs new file mode 100644 index 000000000..b73ba6ead --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.Revision == 0) + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs new file mode 100644 index 000000000..ebb403ba9 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.Revision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs new file mode 100644 index 000000000..296b73779 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs new file mode 100644 index 000000000..296b73779 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs new file mode 100644 index 000000000..296b73779 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs new file mode 100644 index 000000000..296b73779 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs @@ -0,0 +1,184 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs new file mode 100644 index 000000000..3ab3f9d3f --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs new file mode 100644 index 000000000..e4be91c25 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs new file mode 100644 index 000000000..91a738627 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs @@ -0,0 +1,175 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs new file mode 100644 index 000000000..e4eba1d38 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs @@ -0,0 +1,174 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs new file mode 100644 index 000000000..b4dde802b --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs @@ -0,0 +1,175 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); + + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs new file mode 100644 index 000000000..0eae7ff8b --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs @@ -0,0 +1,174 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs new file mode 100644 index 000000000..6bc79f5bd --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs new file mode 100644 index 000000000..6bc79f5bd --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs new file mode 100644 index 000000000..8538f7d87 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await G(engineetBuiltEngineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs new file mode 100644 index 000000000..6bc79f5bd --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs new file mode 100644 index 000000000..67a1023da --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngine(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs new file mode 100644 index 000000000..2cfccdc7e --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs new file mode 100644 index 000000000..2cfccdc7e --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs @@ -0,0 +1,173 @@ +namespace SIL.Machine.AspNetCore.Services; + +public class SmtTransferEngineService : TranslationEngineServiceBase +{ + private readonly IRepository _trainSegmentPairs; + private readonly SmtTransferEngineStateService _stateService; + + public SmtTransferEngineService( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines, + IRepository trainSegmentPairs, + SmtTransferEngineStateService stateService + ) + : base(jobClient, lockFactory, platformService, dataAccessContext, engines) + { + _trainSegmentPairs = trainSegmentPairs; + _stateService = stateService; + } + + public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; + + public override async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); + + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + state.InitNew(); + } + } + + public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await base.DeleteAsync(engineId, cancellationToken); + if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); + await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) + { + // ensure that there is no build running before unloading + string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); + if (buildId is not null) + await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); + + await state.DeleteDataAsync(); + await state.DisposeAsync(); + } + } + } + + public override async Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return results; + } + } + + public override async Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); + state.LastUsedTime = DateTime.Now; + return result; + } + } + + public override async Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState is BuildState.Active) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await _trainSegmentPairs.InsertAsync( + new TrainSegmentPair + { + TranslationEngineRef = engine.Id, + Source = sourceSegment, + Target = targetSegment, + SentenceStart = sentenceStart + }, + cancellationToken + ); + } + + HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); + await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); + await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); + if (engine.BuildState is BuildState.Active) + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + state.IsUpdated = true; + state.LastUsedTime = DateTime.Now; + } + } + + public override async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + SmtTransferEngineState state = _stateService.Get(engineId); + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + state.LastUsedTime = DateTime.UtcNow; + } + } + + protected override Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ) + { + // Token "None" is used here because hangfire injects the proper cancellation token + return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs new file mode 100644 index 000000000..0f3d710af --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs @@ -0,0 +1,212 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs new file mode 100644 index 000000000..41ff71d4b --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs @@ -0,0 +1,219 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) + + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs new file mode 100644 index 000000000..d7acd2bfd --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs @@ -0,0 +1,219 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs new file mode 100644 index 000000000..2d7e5e928 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs @@ -0,0 +1,227 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs new file mode 100644 index 000000000..2d7e5e928 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs @@ -0,0 +1,227 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs new file mode 100644 index 000000000..2d7e5e928 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs @@ -0,0 +1,227 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs new file mode 100644 index 000000000..2d7e5e928 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs @@ -0,0 +1,227 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs new file mode 100644 index 000000000..8c2e0cbc3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs @@ -0,0 +1,230 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task ThrowExceptionIfEngineIsNotBuilt( + string engineId, + CancellationToken cancellationToken + ) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs new file mode 100644 index 000000000..f310e85f3 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs @@ -0,0 +1,227 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs new file mode 100644 index 000000000..dc702f8de --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs @@ -0,0 +1,228 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs new file mode 100644 index 000000000..dc702f8de --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs @@ -0,0 +1,228 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs new file mode 100644 index 000000000..d6511bf11 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs @@ -0,0 +1,228 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs new file mode 100644 index 000000000..dc702f8de --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs @@ -0,0 +1,228 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs new file mode 100644 index 000000000..dc702f8de --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs @@ -0,0 +1,228 @@ +namespace SIL.Machine.AspNetCore.Services; + +public abstract class TranslationEngineServiceBase : ITranslationEngineService +{ + private readonly IBackgroundJobClient _jobClient; + + protected TranslationEngineServiceBase( + IBackgroundJobClient jobClient, + IDistributedReaderWriterLockFactory lockFactory, + IPlatformService platformService, + IDataAccessContext dataAccessContext, + IRepository engines + ) + { + _jobClient = jobClient; + LockFactory = lockFactory; + PlatformService = platformService; + DataAccessContext = dataAccessContext; + Engines = engines; + } + + protected IRepository Engines { get; } + protected IDistributedReaderWriterLockFactory LockFactory { get; } + protected IPlatformService PlatformService { get; } + protected IDataAccessContext DataAccessContext { get; } + + public abstract TranslationEngineType Type { get; } + + public virtual async Task CreateAsync( + string engineId, + string? engineName, + string sourceLanguage, + string targetLanguage, + CancellationToken cancellationToken = default + ) + { + await Engines.InsertAsync( + new TranslationEngine + { + EngineId = engineId, + SourceLanguage = sourceLanguage, + TargetLanguage = targetLanguage + }, + cancellationToken + ); + } + + public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); + await LockFactory.DeleteAsync(engineId, cancellationToken); + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + } + + public virtual async Task StartBuildAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken = default + ) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); + } + } + + public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) + { + IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); + await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) + { + await CancelBuildInternalAsync(engineId, cancellationToken); + } + } + + public virtual Task> TranslateAsync( + string engineId, + int n, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task GetWordGraphAsync( + string engineId, + string segment, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + public virtual Task TrainSegmentPairAsync( + string engineId, + string sourceSegment, + string targetSegment, + bool sentenceStart, + CancellationToken cancellationToken = default + ) + { + throw new NotSupportedException(); + } + + protected abstract Expression> GetJobExpression( + string engineId, + string buildId, + IReadOnlyList corpora + ); + + protected async Task StartBuildInternalAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // If there is a pending job, then no need to start a new one. + if ( + await Engines.ExistsAsync( + e => + e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), + cancellationToken + ) + ) + throw new InvalidOperationException("Engine is already building or pending."); + + // Schedule the job to occur way in the future, just so we can get the job id. + string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); + try + { + await Engines.UpdateAsync( + e => e.EngineId == engineId, + u => + u.Set(e => e.BuildState, BuildState.Pending) + .Set(e => e.IsCanceled, false) + .Set(e => e.JobId, jobId) + .Set(e => e.BuildId, buildId), + cancellationToken: CancellationToken.None + ); + // Enqueue the job now that the build has been created. + _jobClient.Requeue(jobId); + } + catch + { + _jobClient.Delete(jobId); + throw; + } + } + + protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) + { + await DataAccessContext.BeginTransactionAsync(cancellationToken); + // First, try to cancel a job that hasn't started yet + TranslationEngine? engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Pending, + u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), + cancellationToken: cancellationToken + ); + bool notifyPlatform = false; + if (engine is not null) + { + notifyPlatform = true; + } + else + { + // Second, try to cancel a job that is already running + engine = await Engines.UpdateAsync( + e => e.EngineId == engineId && e.BuildState == BuildState.Active, + u => u.Set(b => b.IsCanceled, true), + cancellationToken: cancellationToken + ); + } + if (engine is not null) + { + // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token + _jobClient.Delete(engine.JobId); + if (notifyPlatform) + await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); + } + await DataAccessContext.CommitTransactionAsync(CancellationToken.None); + return engine?.BuildId; + } + + protected async Task WaitForBuildToFinishAsync( + string engineId, + string buildId, + CancellationToken cancellationToken + ) + { + using ISubscription sub = await Engines.SubscribeAsync( + e => e.EngineId == engineId && e.BuildId == buildId, + cancellationToken + ); + if (sub.Change.Entity is null) + return true; + + var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); + while (DateTime.UtcNow < timeout) + { + await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); + TranslationEngine? engine = sub.Change.Entity; + if (engine is null || engine.BuildState is BuildState.None) + return true; + } + return false; + } + + protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); + if (engine is null) + throw new InvalidOperationException(""); + return engine; + } + + protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) + { + TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); + if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere + throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); + return engine; + } +} diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs new file mode 100644 index 000000000..21868db71 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs @@ -0,0 +1,47 @@ +global using System.Collections; +global using System.Collections.Concurrent; +global using System.Diagnostics; +global using System.Diagnostics.CodeAnalysis; +global using System.IO.Compression; +global using System.Linq.Expressions; +global using System.Net; +global using System.Reflection; +global using System.Runtime.CompilerServices; +global using System.Text; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Amazon; +global using Amazon.S3; +global using Amazon.S3.Model; +global using Amazon.S3.Transfer; +global using Grpc.Core; +global using Grpc.Core.Interceptors; +global using Grpc.Net.Client.Configuration; +global using Hangfire; +global using Hangfire.Mongo; +global using Hangfire.Mongo.Migration.Strategies; +global using Hangfire.Mongo.Migration.Strategies.Backup; +global using Microsoft.AspNetCore.Routing; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Diagnostics.HealthChecks; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using MongoDB.Driver; +global using MongoDB.Driver.Linq; +global using Nito.AsyncEx; +global using Polly; +global using SIL.DataAccess; +global using SIL.Machine.AspNetCore.Configuration; +global using SIL.Machine.AspNetCore.Models; +global using SIL.Machine.AspNetCore.Services; +global using SIL.Machine.AspNetCore.Utils; +global using SIL.Machine.Corpora; +global using SIL.Machine.Morphology.HermitCrab; +global using SIL.Machine.Tokenization; +global using SIL.Machine.Translation; +global using SIL.Machine.Translation.Thot; +global using SIL.Machine.Utils; +global using SIL.ObjectModel; +global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs new file mode 100644 index 000000000..0b07fae03 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs @@ -0,0 +1,47 @@ +global using System.Collections; +global using System.Collections.Concurrent; +global using System.Diagnostics; +global using System.Diagnostics.CodeAnalysis; +global using System.IO.Compression; +global using System.Linq.Expressions; +global using System.Net; +global using System.Reflection; +global using System.Runtime.CompilerServices; +global using System.Text; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Amazon; +global using Amazon.S3; +global using Amazon.S3.Model; +global using Amazon.Runtime; +global using Grpc.Core; +global using Grpc.Core.Interceptors; +global using Grpc.Net.Client.Configuration; +global using Hangfire; +global using Hangfire.Mongo; +global using Hangfire.Mongo.Migration.Strategies; +global using Hangfire.Mongo.Migration.Strategies.Backup; +global using Microsoft.AspNetCore.Routing; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Diagnostics.HealthChecks; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using MongoDB.Driver; +global using MongoDB.Driver.Linq; +global using Nito.AsyncEx; +global using Polly; +global using SIL.DataAccess; +global using SIL.Machine.AspNetCore.Configuration; +global using SIL.Machine.AspNetCore.Models; +global using SIL.Machine.AspNetCore.Services; +global using SIL.Machine.AspNetCore.Utils; +global using SIL.Machine.Corpora; +global using SIL.Machine.Morphology.HermitCrab; +global using SIL.Machine.Tokenization; +global using SIL.Machine.Translation; +global using SIL.Machine.Translation.Thot; +global using SIL.Machine.Utils; +global using SIL.ObjectModel; +global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs new file mode 100644 index 000000000..a808510e6 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs @@ -0,0 +1,49 @@ +global using System.Collections; +global using System.Collections.Concurrent; +global using System.Diagnostics; +global using System.Diagnostics.CodeAnalysis; +global using System.IO.Compression; +global using System.Linq.Expressions; +global using System.Net; +global using System.Reflection; +global using System.Runtime.CompilerServices; +global using System.Text; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Amazon; +global using Amazon.S3; +global using Amazon.S3.Model; +global using Amazon.Runtime; +global using Grpc.Core; +global using Grpc.Core.Interceptors; +global using Grpc.Net.Client.Configuration; +global using Hangfire; +global using Hangfire.Mongo; +global using Hangfire.Mongo.Migration.Strategies; +global using Hangfire.Mongo.Migration.Strategies.Backup; +global using Microsoft.AspNetCore.Routing; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Diagnostics.HealthChecks; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using MongoDB.Driver; +global using MongoDB.Driver.Linq; +global using Nito.AsyncEx; +using Nito.AsyncEx.Synchronous; + +global using Polly; +global using SIL.DataAccess; +global using SIL.Machine.AspNetCore.Configuration; +global using SIL.Machine.AspNetCore.Models; +global using SIL.Machine.AspNetCore.Services; +global using SIL.Machine.AspNetCore.Utils; +global using SIL.Machine.Corpora; +global using SIL.Machine.Morphology.HermitCrab; +global using SIL.Machine.Tokenization; +global using SIL.Machine.Translation; +global using SIL.Machine.Translation.Thot; +global using SIL.Machine.Utils; +global using SIL.ObjectModel; +global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs new file mode 100644 index 000000000..981c59ef5 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs @@ -0,0 +1,48 @@ +global using System.Collections; +global using System.Collections.Concurrent; +global using System.Diagnostics; +global using System.Diagnostics.CodeAnalysis; +global using System.IO.Compression; +global using System.Linq.Expressions; +global using System.Net; +global using System.Reflection; +global using System.Runtime.CompilerServices; +global using System.Text; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Amazon; +global using Amazon.S3; +global using Amazon.S3.Model; +global using Amazon.Runtime; +global using Grpc.Core; +global using Grpc.Core.Interceptors; +global using Grpc.Net.Client.Configuration; +global using Hangfire; +global using Hangfire.Mongo; +global using Hangfire.Mongo.Migration.Strategies; +global using Hangfire.Mongo.Migration.Strategies.Backup; +global using Microsoft.AspNetCore.Routing; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Diagnostics.HealthChecks; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using MongoDB.Driver; +global using MongoDB.Driver.Linq; +global using Nito.AsyncEx; +using Nito.AsyncEx.Synchronous; +global using Polly; +global using SIL.DataAccess; +global using SIL.Machine.AspNetCore.Configuration; +global using SIL.Machine.AspNetCore.Models; +global using SIL.Machine.AspNetCore.Services; +global using SIL.Machine.AspNetCore.Utils; +global using SIL.Machine.Corpora; +global using SIL.Machine.Morphology.HermitCrab; +global using SIL.Machine.Tokenization; +global using SIL.Machine.Translation; +global using SIL.Machine.Translation.Thot; +global using SIL.Machine.Utils; +global using SIL.ObjectModel; +global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs new file mode 100644 index 000000000..8544d1e96 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs @@ -0,0 +1,48 @@ +global using System.Collections; +global using System.Collections.Concurrent; +global using System.Diagnostics; +global using System.Diagnostics.CodeAnalysis; +global using System.IO.Compression; +global using System.Linq.Expressions; +global using System.Net; +global using System.Reflection; +global using System.Runtime.CompilerServices; +global using System.Text; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Amazon; +global using Amazon.S3; +global using Amazon.S3.Model; +global using Amazon.Runtime; +global using Grpc.Core; +global using Grpc.Core.Interceptors; +global using Grpc.Net.Client.Configuration; +global using Hangfire; +global using Hangfire.Mongo; +global using Hangfire.Mongo.Migration.Strategies; +global using Hangfire.Mongo.Migration.Strategies.Backup; +global using Microsoft.AspNetCore.Routing; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Diagnostics.HealthChecks; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using MongoDB.Driver; +global using MongoDB.Driver.Linq; +global using Nito.AsyncEx; +global using Nito.AsyncEx.Synchronous; +global using Polly; +global using SIL.DataAccess; +global using SIL.Machine.AspNetCore.Configuration; +global using SIL.Machine.AspNetCore.Models; +global using SIL.Machine.AspNetCore.Services; +global using SIL.Machine.AspNetCore.Utils; +global using SIL.Machine.Corpora; +global using SIL.Machine.Morphology.HermitCrab; +global using SIL.Machine.Tokenization; +global using SIL.Machine.Translation; +global using SIL.Machine.Translation.Thot; +global using SIL.Machine.Utils; +global using SIL.ObjectModel; +global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs new file mode 100644 index 000000000..8544d1e96 --- /dev/null +++ b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs @@ -0,0 +1,48 @@ +global using System.Collections; +global using System.Collections.Concurrent; +global using System.Diagnostics; +global using System.Diagnostics.CodeAnalysis; +global using System.IO.Compression; +global using System.Linq.Expressions; +global using System.Net; +global using System.Reflection; +global using System.Runtime.CompilerServices; +global using System.Text; +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Amazon; +global using Amazon.S3; +global using Amazon.S3.Model; +global using Amazon.Runtime; +global using Grpc.Core; +global using Grpc.Core.Interceptors; +global using Grpc.Net.Client.Configuration; +global using Hangfire; +global using Hangfire.Mongo; +global using Hangfire.Mongo.Migration.Strategies; +global using Hangfire.Mongo.Migration.Strategies.Backup; +global using Microsoft.AspNetCore.Routing; +global using Microsoft.Extensions.Configuration; +global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Diagnostics.HealthChecks; +global using Microsoft.Extensions.Hosting; +global using Microsoft.Extensions.Logging; +global using Microsoft.Extensions.Options; +global using MongoDB.Driver; +global using MongoDB.Driver.Linq; +global using Nito.AsyncEx; +global using Nito.AsyncEx.Synchronous; +global using Polly; +global using SIL.DataAccess; +global using SIL.Machine.AspNetCore.Configuration; +global using SIL.Machine.AspNetCore.Models; +global using SIL.Machine.AspNetCore.Services; +global using SIL.Machine.AspNetCore.Utils; +global using SIL.Machine.Corpora; +global using SIL.Machine.Morphology.HermitCrab; +global using SIL.Machine.Tokenization; +global using SIL.Machine.Translation; +global using SIL.Machine.Translation.Thot; +global using SIL.Machine.Utils; +global using SIL.ObjectModel; +global using SIL.WritingSystems; diff --git a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs b/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs new file mode 100644 index 000000000..f9d01a7d6 --- /dev/null +++ b/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs @@ -0,0 +1,192 @@ +namespace SIL.Machine.AspNetCore.Services; + +[TestFixture] +public class ClearMLNmtEngineServiceTests +{ + [Test] + public async Task CancelBuildAsync() + { + using var env = new TestEnvironment(); + env.ClearMLService + .CreateTaskAsync( + Arg.Any(), + "project1", + "engine1", + "es", + "en", + "memory:///", + Arg.Any() + ) + .Returns(Task.FromResult("task1")); + var task = new ClearMLTask + { + Id = "task1", + Project = new ClearMLProject { Id = "project1" }, + Status = ClearMLTaskStatus.InProgress + }; + bool first = true; + env.ClearMLService + .GetTaskByNameAsync(Arg.Any(), Arg.Any()) + .Returns(x => + { + if (first) + { + first = false; + return Task.FromResult(null); + } + return Task.FromResult(task); + }); + env.ClearMLService + .GetTaskByIdAsync("task1", Arg.Any()) + .Returns(Task.FromResult(task)); + await env.Service.StartBuildAsync("engine1", "build1", Array.Empty()); + await env.WaitForBuildToStartAsync(); + TranslationEngine engine = env.Engines.Get("engine1"); + Assert.That(engine.BuildState, Is.EqualTo(BuildState.Active)); + await env.Service.CancelBuildAsync("engine1"); + await env.WaitForBuildToFinishAsync(); + engine = env.Engines.Get("engine1"); + Assert.That(engine.BuildState, Is.EqualTo(BuildState.None)); + await env.ClearMLService.Received().StopTaskAsync("task1", Arg.Any()); + } + + private class TestEnvironment : DisposableBase + { + private readonly MemoryStorage _memoryStorage; + private readonly BackgroundJobClient _jobClient; + private BackgroundJobServer _jobServer; + private readonly IDistributedReaderWriterLockFactory _lockFactory; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + + public TestEnvironment() + { + Engines = new MemoryRepository(); + Engines.Add( + new TranslationEngine + { + Id = "engine1", + EngineId = "engine1", + SourceLanguage = "es", + TargetLanguage = "en" + } + ); + EngineOptions = new SmtTransferEngineOptions(); + _memoryStorage = new MemoryStorage(); + _jobClient = new BackgroundJobClient(_memoryStorage); + PlatformService = Substitute.For(); + ClearMLService = Substitute.For(); + ClearMLService + .GetProjectIdAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult("project1")); + _lockFactory = new DistributedReaderWriterLockFactory( + new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), + new MemoryRepository(), + new ObjectIdGenerator() + ); + _sharedFileService = new SharedFileService(); + _options = Substitute.For>(); + _options.CurrentValue.Returns( + new ClearMLNmtEngineOptions { BuildPollingTimeout = TimeSpan.FromMilliseconds(50) } + ); + _jobServer = CreateJobServer(); + Service = CreateService(); + } + + public ClearMLNmtEngineService Service { get; private set; } + public MemoryRepository Engines { get; } + public SmtTransferEngineOptions EngineOptions { get; } + public IPlatformService PlatformService { get; } + public IClearMLService ClearMLService { get; } + + public void StopServer() + { + _jobServer.Dispose(); + } + + public void StartServer() + { + _jobServer = CreateJobServer(); + Service = CreateService(); + } + + private BackgroundJobServer CreateJobServer() + { + var jobServerOptions = new BackgroundJobServerOptions + { + Activator = new EnvActivator(this), + Queues = new[] { "nmt" }, + CancellationCheckInterval = TimeSpan.FromMilliseconds(100), + }; + return new BackgroundJobServer(jobServerOptions, _memoryStorage); + } + + private ClearMLNmtEngineService CreateService() + { + return new ClearMLNmtEngineService( + _jobClient, + PlatformService, + _lockFactory, + new MemoryDataAccessContext(), + Engines, + ClearMLService + ); + } + + public Task WaitForBuildToFinishAsync() + { + return WaitForBuildState(e => e.BuildState is BuildState.None); + } + + public Task WaitForBuildToStartAsync() + { + return WaitForBuildState(e => e.BuildState is BuildState.Active); + } + + private async Task WaitForBuildState(Func predicate) + { + using ISubscription subscription = await Engines.SubscribeAsync( + e => e.EngineId == "engine1" + ); + while (true) + { + TranslationEngine? build = subscription.Change.Entity; + if (build is not null && predicate(build)) + break; + await subscription.WaitForChangeAsync(); + } + } + + protected override void DisposeManagedResources() + { + _jobServer.Dispose(); + } + + private class EnvActivator : JobActivator + { + private readonly TestEnvironment _env; + + public EnvActivator(TestEnvironment env) + { + _env = env; + } + + public override object ActivateJob(Type jobType) + { + if (jobType == typeof(ClearMLNmtEngineBuildJob)) + { + return new ClearMLNmtEngineBuildJob( + _env.PlatformService, + _env.Engines, + Substitute.For>(), + _env.ClearMLService, + _env._sharedFileService, + _env._options, + Substitute.For() + ); + } + return base.ActivateJob(jobType); + } + } + } +} diff --git a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs b/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs new file mode 100644 index 000000000..614dabc52 --- /dev/null +++ b/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs @@ -0,0 +1,192 @@ +namespace SIL.Machine.AspNetCore.Services; + +[TestFixture] +public class ClearMLNmtEngineServiceTests +{ + [Test] + public async Task CancelBuildAsync() + { + using var env = new TestEnvironment(); + env.ClearMLService + .CreateTaskAsync( + Arg.Any(), + "project1", + "engine1", + "es", + "en", + "memory:///", + Arg.Any() + ) + .Returns(Task.FromResult("task1")); + var task = new ClearMLTask + { + Id = "task1", + Project = new ClearMLProject { Id = "project1" }, + Status = ClearMLTaskStatus.InProgress + }; + bool first = true; + env.ClearMLService + .GetTaskByNameAsync(Arg.Any(), Arg.Any()) + .Returns(x => + { + if (first) + { + first = false; + return Task.FromResult(null); + } + return Task.FromResult(task); + }); + env.ClearMLService + .GetTaskByIdAsync("task1", Arg.Any()) + .Returns(Task.FromResult(task)); + await env.Service.StartBuildAsync("engine1", "build1", Array.Empty()); + await env.WaitForBuildToStartAsync(); + TranslationEngine engine = env.Engines.Get("engine1"); + Assert.That(engine.BuildState, Is.EqualTo(BuildState.Active)); + await env.Service.CancelBuildAsync("engine1"); + await env.WaitForBuildToFinishAsync(); + engine = env.Engines.Get("engine1"); + Assert.That(engine.BuildState, Is.EqualTo(BuildState.None)); + await env.ClearMLService.Received().StopTaskAsync("task1", Arg.Any()); + } + + private class TestEnvironment : DisposableBase + { + private readonly MemoryStorage _memoryStorage; + private readonly BackgroundJobClient _jobClient; + private BackgroundJobServer _jobServer; + private readonly IDistributedReaderWriterLockFactory _lockFactory; + private readonly ISharedFileService _sharedFileService; + private readonly IOptionsMonitor _options; + + public TestEnvironment() + { + Engines = new MemoryRepository(); + Engines.Add( + new TranslationEngine + { + Id = "engine1", + EngineId = "engine1", + SourceLanguage = "es", + TargetLanguage = "en" + } + ); + EngineOptions = new SmtTransferEngineOptions(); + _memoryStorage = new MemoryStorage(); + _jobClient = new BackgroundJobClient(_memoryStorage); + PlatformService = Substitute.For(); + ClearMLService = Substitute.For(); + ClearMLService + .GetProjectIdAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult("project1")); + _lockFactory = new DistributedReaderWriterLockFactory( + new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), + new MemoryRepository(), + new ObjectIdGenerator() + ); + _sharedFileService = new SharedFileService(Substitute.For()); + _options = Substitute.For>(); + _options.CurrentValue.Returns( + new ClearMLNmtEngineOptions { BuildPollingTimeout = TimeSpan.FromMilliseconds(50) } + ); + _jobServer = CreateJobServer(); + Service = CreateService(); + } + + public ClearMLNmtEngineService Service { get; private set; } + public MemoryRepository Engines { get; } + public SmtTransferEngineOptions EngineOptions { get; } + public IPlatformService PlatformService { get; } + public IClearMLService ClearMLService { get; } + + public void StopServer() + { + _jobServer.Dispose(); + } + + public void StartServer() + { + _jobServer = CreateJobServer(); + Service = CreateService(); + } + + private BackgroundJobServer CreateJobServer() + { + var jobServerOptions = new BackgroundJobServerOptions + { + Activator = new EnvActivator(this), + Queues = new[] { "nmt" }, + CancellationCheckInterval = TimeSpan.FromMilliseconds(100), + }; + return new BackgroundJobServer(jobServerOptions, _memoryStorage); + } + + private ClearMLNmtEngineService CreateService() + { + return new ClearMLNmtEngineService( + _jobClient, + PlatformService, + _lockFactory, + new MemoryDataAccessContext(), + Engines, + ClearMLService + ); + } + + public Task WaitForBuildToFinishAsync() + { + return WaitForBuildState(e => e.BuildState is BuildState.None); + } + + public Task WaitForBuildToStartAsync() + { + return WaitForBuildState(e => e.BuildState is BuildState.Active); + } + + private async Task WaitForBuildState(Func predicate) + { + using ISubscription subscription = await Engines.SubscribeAsync( + e => e.EngineId == "engine1" + ); + while (true) + { + TranslationEngine? build = subscription.Change.Entity; + if (build is not null && predicate(build)) + break; + await subscription.WaitForChangeAsync(); + } + } + + protected override void DisposeManagedResources() + { + _jobServer.Dispose(); + } + + private class EnvActivator : JobActivator + { + private readonly TestEnvironment _env; + + public EnvActivator(TestEnvironment env) + { + _env = env; + } + + public override object ActivateJob(Type jobType) + { + if (jobType == typeof(ClearMLNmtEngineBuildJob)) + { + return new ClearMLNmtEngineBuildJob( + _env.PlatformService, + _env.Engines, + Substitute.For>(), + _env.ClearMLService, + _env._sharedFileService, + _env._options, + Substitute.For() + ); + } + return base.ActivateJob(jobType); + } + } + } +} From e8903168c3b3ab325d647b378257d7e52d086e24 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Mon, 11 Sep 2023 13:05:35 -0400 Subject: [PATCH 10/10] Revert "Empty commit to trigger workflow --ECL" This reverts commit c9384c86e4886efa63fa416e76571f0c88b7f0e6. --- .history/.gitignore_20230911115750 | 51 --- .history/.gitignore_20230911115828 | 52 --- ...MachineBuilderExtensions_20230907084316.cs | 317 ---------------- ...MachineBuilderExtensions_20230907133310.cs | 317 ---------------- ...ClearMLNmtEngineBuildJob_20230907084316.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907095831.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907135725.cs | 343 ------------------ ...ClearMLNmtEngineBuildJob_20230907135737.cs | 342 ----------------- ...ClearMLNmtEngineBuildJob_20230907135738.cs | 342 ----------------- ...ClearMLNmtEngineBuildJob_20230907135739.cs | 342 ----------------- ...ClearMLNmtEngineBuildJob_20230907140302.cs | 342 ----------------- ...ClearMLNmtEngineBuildJob_20230907140611.cs | 342 ----------------- ...ClearMLNmtEngineBuildJob_20230907140709.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907140719.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907140720.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907141110.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907141112.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907141905.cs | 340 ----------------- .../ClearMLNmtEngineService_20230906141540.cs | 51 --- .../ClearMLNmtEngineService_20230911085826.cs | 51 --- .../ClearMLNmtEngineService_20230911085829.cs | 51 --- .../Services/S3FileStorage_20230906141540.cs | 95 ----- .../Services/S3FileStorage_20230907104119.cs | 95 ----- .../Services/S3FileStorage_20230907130153.cs | 97 ----- .../Services/S3FileStorage_20230907130157.cs | 97 ----- .../Services/S3FileStorage_20230907132651.cs | 98 ----- .../Services/S3FileStorage_20230907132652.cs | 98 ----- .../Services/S3FileStorage_20230907171652.cs | 98 ----- .../Services/S3FileStorage_20230907171653.cs | 98 ----- .../Services/S3FileStorage_20230907171816.cs | 107 ------ .../Services/S3FileStorage_20230907171821.cs | 107 ------ .../Services/S3FileStorage_20230907171829.cs | 108 ------ .../Services/S3FileStorage_20230907171837.cs | 107 ------ .../Services/S3FileStorage_20230907171838.cs | 107 ------ .../Services/S3FileStorage_20230908125336.cs | 107 ------ .../Services/S3WriteStream_20230906141540.cs | 76 ---- .../Services/S3WriteStream_20230907104130.cs | 76 ---- .../Services/S3WriteStream_20230907104133.cs | 76 ---- .../Services/S3WriteStream_20230907130227.cs | 77 ---- .../Services/S3WriteStream_20230907130326.cs | 79 ---- .../Services/S3WriteStream_20230907130328.cs | 78 ---- .../Services/S3WriteStream_20230907130334.cs | 78 ---- .../Services/S3WriteStream_20230907130427.cs | 87 ----- .../Services/S3WriteStream_20230907130442.cs | 75 ---- .../Services/S3WriteStream_20230907130507.cs | 108 ------ .../Services/S3WriteStream_20230907130523.cs | 108 ------ .../Services/S3WriteStream_20230907130903.cs | 106 ------ .../Services/S3WriteStream_20230907130942.cs | 107 ------ .../Services/S3WriteStream_20230907130949.cs | 108 ------ .../Services/S3WriteStream_20230907131020.cs | 108 ------ .../Services/S3WriteStream_20230907131032.cs | 108 ------ .../Services/S3WriteStream_20230907131048.cs | 109 ------ .../Services/S3WriteStream_20230907131526.cs | 110 ------ .../Services/S3WriteStream_20230907131529.cs | 113 ------ .../Services/S3WriteStream_20230907131704.cs | 116 ------ .../Services/S3WriteStream_20230907131835.cs | 114 ------ .../Services/S3WriteStream_20230907131840.cs | 113 ------ .../Services/S3WriteStream_20230907131855.cs | 114 ------ .../Services/S3WriteStream_20230907131914.cs | 115 ------ .../Services/S3WriteStream_20230907131918.cs | 115 ------ .../Services/S3WriteStream_20230907132115.cs | 120 ------ .../Services/S3WriteStream_20230907132117.cs | 120 ------ .../Services/S3WriteStream_20230907132230.cs | 120 ------ .../Services/S3WriteStream_20230907132231.cs | 120 ------ .../Services/S3WriteStream_20230907132240.cs | 120 ------ .../Services/S3WriteStream_20230907132242.cs | 120 ------ .../Services/S3WriteStream_20230907132338.cs | 120 ------ .../Services/S3WriteStream_20230907132350.cs | 125 ------- .../Services/S3WriteStream_20230907132400.cs | 125 ------- .../Services/S3WriteStream_20230907132417.cs | 125 ------- .../Services/S3WriteStream_20230907132419.cs | 124 ------- .../Services/S3WriteStream_20230907132431.cs | 125 ------- .../Services/S3WriteStream_20230907132432.cs | 124 ------- .../Services/S3WriteStream_20230907132433.cs | 124 ------- .../Services/S3WriteStream_20230907132616.cs | 125 ------- .../Services/S3WriteStream_20230907132630.cs | 125 ------- .../Services/S3WriteStream_20230907132632.cs | 125 ------- .../Services/S3WriteStream_20230907132832.cs | 126 ------- .../Services/S3WriteStream_20230907132840.cs | 126 ------- .../Services/S3WriteStream_20230907133744.cs | 126 ------- .../Services/S3WriteStream_20230907140811.cs | 127 ------- .../Services/S3WriteStream_20230907140817.cs | 151 -------- .../Services/S3WriteStream_20230907140826.cs | 151 -------- .../Services/S3WriteStream_20230907141040.cs | 155 -------- .../Services/S3WriteStream_20230907141052.cs | 155 -------- .../Services/S3WriteStream_20230907141053.cs | 155 -------- .../Services/S3WriteStream_20230907141054.cs | 155 -------- .../Services/S3WriteStream_20230907141056.cs | 155 -------- .../Services/S3WriteStream_20230907141058.cs | 155 -------- .../Services/S3WriteStream_20230907141059.cs | 155 -------- .../Services/S3WriteStream_20230907141100.cs | 155 -------- .../Services/S3WriteStream_20230907141102.cs | 155 -------- .../Services/S3WriteStream_20230907142255.cs | 153 -------- .../Services/S3WriteStream_20230907171436.cs | 155 -------- .../Services/S3WriteStream_20230907171441.cs | 155 -------- .../Services/S3WriteStream_20230907171450.cs | 155 -------- .../Services/S3WriteStream_20230907171526.cs | 155 -------- .../Services/S3WriteStream_20230907171534.cs | 155 -------- .../Services/S3WriteStream_20230907171604.cs | 155 -------- .../Services/S3WriteStream_20230907171626.cs | 155 -------- .../Services/S3WriteStream_20230907171627.cs | 155 -------- .../Services/S3WriteStream_20230907171628.cs | 155 -------- .../Services/S3WriteStream_20230907171630.cs | 155 -------- .../Services/S3WriteStream_20230907171636.cs | 155 -------- .../Services/S3WriteStream_20230907171638.cs | 155 -------- .../Services/S3WriteStream_20230907171720.cs | 161 -------- .../Services/S3WriteStream_20230907171734.cs | 161 -------- .../Services/S3WriteStream_20230907171736.cs | 161 -------- .../Services/S3WriteStream_20230907172014.cs | 162 --------- .../Services/S3WriteStream_20230907172018.cs | 163 --------- .../Services/S3WriteStream_20230907172019.cs | 163 --------- .../Services/S3WriteStream_20230907172038.cs | 163 --------- .../Services/S3WriteStream_20230907172039.cs | 163 --------- .../Services/S3WriteStream_20230908125336.cs | 163 --------- .../Services/S3WriteStream_20230908125337.cs | 163 --------- .../Services/S3WriteStream_20230908125344.cs | 163 --------- .../SharedFileService_20230906141540.cs | 92 ----- .../SharedFileService_20230907104153.cs | 92 ----- .../SharedFileService_20230907134349.cs | 92 ----- .../SharedFileService_20230907134355.cs | 92 ----- .../SharedFileService_20230907171909.cs | 95 ----- .../SharedFileService_20230907171920.cs | 97 ----- .../SharedFileService_20230907171930.cs | 98 ----- .../SharedFileService_20230907171931.cs | 98 ----- .../SharedFileService_20230907171955.cs | 98 ----- .../SharedFileService_20230907171956.cs | 98 ----- .../SharedFileService_20230908125406.cs | 96 ----- .../SharedFileService_20230908125407.cs | 96 ----- ...SmtTransferEngineService_20230907084301.cs | 181 --------- ...SmtTransferEngineService_20230908161128.cs | 183 ---------- ...SmtTransferEngineService_20230908161129.cs | 183 ---------- ...SmtTransferEngineService_20230908161150.cs | 183 ---------- ...SmtTransferEngineService_20230908161950.cs | 183 ---------- ...SmtTransferEngineService_20230908162047.cs | 184 ---------- ...SmtTransferEngineService_20230908162317.cs | 188 ---------- ...SmtTransferEngineService_20230908162334.cs | 188 ---------- ...SmtTransferEngineService_20230908162336.cs | 188 ---------- ...SmtTransferEngineService_20230908162356.cs | 188 ---------- ...SmtTransferEngineService_20230908162358.cs | 188 ---------- ...SmtTransferEngineService_20230911083245.cs | 190 ---------- ...SmtTransferEngineService_20230911083247.cs | 190 ---------- ...SmtTransferEngineService_20230911083620.cs | 189 ---------- ...SmtTransferEngineService_20230911083625.cs | 189 ---------- ...SmtTransferEngineService_20230911083627.cs | 189 ---------- ...SmtTransferEngineService_20230911083628.cs | 189 ---------- ...SmtTransferEngineService_20230911083939.cs | 190 ---------- ...SmtTransferEngineService_20230911084126.cs | 192 ---------- ...SmtTransferEngineService_20230911084129.cs | 188 ---------- ...SmtTransferEngineService_20230911084135.cs | 188 ---------- ...SmtTransferEngineService_20230911084136.cs | 188 ---------- ...SmtTransferEngineService_20230911084149.cs | 188 ---------- ...SmtTransferEngineService_20230911084150.cs | 188 ---------- ...SmtTransferEngineService_20230911084151.cs | 188 ---------- ...SmtTransferEngineService_20230911084152.cs | 188 ---------- ...SmtTransferEngineService_20230911084153.cs | 188 ---------- ...SmtTransferEngineService_20230911084154.cs | 188 ---------- ...SmtTransferEngineService_20230911084155.cs | 188 ---------- ...SmtTransferEngineService_20230911084156.cs | 188 ---------- ...SmtTransferEngineService_20230911084157.cs | 188 ---------- ...SmtTransferEngineService_20230911084158.cs | 188 ---------- ...SmtTransferEngineService_20230911084159.cs | 188 ---------- ...SmtTransferEngineService_20230911084200.cs | 188 ---------- ...SmtTransferEngineService_20230911084201.cs | 188 ---------- ...SmtTransferEngineService_20230911084202.cs | 188 ---------- ...SmtTransferEngineService_20230911084203.cs | 188 ---------- ...SmtTransferEngineService_20230911084204.cs | 188 ---------- ...SmtTransferEngineService_20230911084205.cs | 188 ---------- ...SmtTransferEngineService_20230911084231.cs | 189 ---------- ...SmtTransferEngineService_20230911084236.cs | 191 ---------- ...SmtTransferEngineService_20230911084237.cs | 190 ---------- ...SmtTransferEngineService_20230911084410.cs | 190 ---------- ...SmtTransferEngineService_20230911084539.cs | 188 ---------- ...SmtTransferEngineService_20230911084556.cs | 185 ---------- ...SmtTransferEngineService_20230911084557.cs | 184 ---------- ...SmtTransferEngineService_20230911084610.cs | 185 ---------- ...SmtTransferEngineService_20230911084658.cs | 184 ---------- ...SmtTransferEngineService_20230911084700.cs | 184 ---------- ...SmtTransferEngineService_20230911084701.cs | 184 ---------- ...SmtTransferEngineService_20230911084711.cs | 184 ---------- ...SmtTransferEngineService_20230911084713.cs | 184 ---------- ...SmtTransferEngineService_20230911084838.cs | 184 ---------- ...SmtTransferEngineService_20230911084843.cs | 184 ---------- ...SmtTransferEngineService_20230911084844.cs | 184 ---------- ...SmtTransferEngineService_20230911084845.cs | 184 ---------- ...SmtTransferEngineService_20230911084846.cs | 184 ---------- ...SmtTransferEngineService_20230911085524.cs | 173 --------- ...SmtTransferEngineService_20230911085526.cs | 173 --------- ...SmtTransferEngineService_20230911085550.cs | 175 --------- ...SmtTransferEngineService_20230911085551.cs | 174 --------- ...SmtTransferEngineService_20230911085557.cs | 175 --------- ...SmtTransferEngineService_20230911085558.cs | 174 --------- ...SmtTransferEngineService_20230911085704.cs | 173 --------- ...SmtTransferEngineService_20230911085706.cs | 173 --------- ...SmtTransferEngineService_20230911085712.cs | 173 --------- ...SmtTransferEngineService_20230911085757.cs | 173 --------- ...SmtTransferEngineService_20230911085803.cs | 173 --------- ...SmtTransferEngineService_20230911085805.cs | 173 --------- ...SmtTransferEngineService_20230911085809.cs | 173 --------- ...slationEngineServiceBase_20230907084301.cs | 212 ----------- ...slationEngineServiceBase_20230911085214.cs | 219 ----------- ...slationEngineServiceBase_20230911085221.cs | 219 ----------- ...slationEngineServiceBase_20230911085339.cs | 227 ------------ ...slationEngineServiceBase_20230911085341.cs | 227 ------------ ...slationEngineServiceBase_20230911085421.cs | 227 ------------ ...slationEngineServiceBase_20230911085505.cs | 227 ------------ ...slationEngineServiceBase_20230911085623.cs | 230 ------------ ...slationEngineServiceBase_20230911085631.cs | 227 ------------ ...slationEngineServiceBase_20230911085641.cs | 228 ------------ ...slationEngineServiceBase_20230911085646.cs | 228 ------------ ...slationEngineServiceBase_20230911085651.cs | 228 ------------ ...slationEngineServiceBase_20230911085722.cs | 228 ------------ ...slationEngineServiceBase_20230911085823.cs | 228 ------------ .../Usings_20230907084316.cs | 47 --- .../Usings_20230907131743.cs | 47 --- .../Usings_20230907141855.cs | 49 --- .../Usings_20230907141856.cs | 48 --- .../Usings_20230907141900.cs | 48 --- .../Usings_20230907141901.cs | 48 --- ...rMLNmtEngineServiceTests_20230906141540.cs | 192 ---------- ...rMLNmtEngineServiceTests_20230907185104.cs | 192 ---------- 220 files changed, 35526 deletions(-) delete mode 100644 .history/.gitignore_20230911115750 delete mode 100644 .history/.gitignore_20230911115828 delete mode 100644 .history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs delete mode 100644 .history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs delete mode 100644 .history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs diff --git a/.history/.gitignore_20230911115750 b/.history/.gitignore_20230911115750 deleted file mode 100644 index e1f430ccb..000000000 --- a/.history/.gitignore_20230911115750 +++ /dev/null @@ -1,51 +0,0 @@ - -#ignore thumbnails created by windows -Thumbs.db -#Ignore files build by Visual Studio -*.obj -*.exe -*.pdb -*.user -*.aps -*.pch -*.vspscc -*_i.c -*_p.c -*.ncb -*.suo -*.tlb -*.tlh -*.bak -*.cache -*.ilk -*.log -[Bb]in -[Dd]ebug*/ -*.lib -*.sbr -obj/ -[Rr]elease*/ -_ReSharper*/ -[Tt]est[Rr]esult* - -*.Resharper -*.DotSettings -*- Copy* -packages/* -!packages/repositories.config -lib/ -*.files -*.VC.db -*.VC.opendb -*.lock.json -.vs -appsettings.user.json -artifacts -!**/Tes/release/ -thot-new-model.zip -*.tgz -out/ -src/sentencepiece4c/build/ -src/sentencepiece4cbuild/ -!samples/**/release -**/*.feature.cs diff --git a/.history/.gitignore_20230911115828 b/.history/.gitignore_20230911115828 deleted file mode 100644 index fd44de4b5..000000000 --- a/.history/.gitignore_20230911115828 +++ /dev/null @@ -1,52 +0,0 @@ - -#ignore thumbnails created by windows -Thumbs.db -#Ignore files build by Visual Studio -*.obj -*.exe -*.pdb -*.user -*.aps -*.pch -*.vspscc -*_i.c -*_p.c -*.ncb -*.suo -*.tlb -*.tlh -*.bak -*.cache -*.ilk -*.log -[Bb]in -[Dd]ebug*/ -*.lib -*.sbr -obj/ -[Rr]elease*/ -_ReSharper*/ -[Tt]est[Rr]esult* -.history* - -*.Resharper -*.DotSettings -*- Copy* -packages/* -!packages/repositories.config -lib/ -*.files -*.VC.db -*.VC.opendb -*.lock.json -.vs -appsettings.user.json -artifacts -!**/Tes/release/ -thot-new-model.zip -*.tgz -out/ -src/sentencepiece4c/build/ -src/sentencepiece4cbuild/ -!samples/**/release -**/*.feature.cs diff --git a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs b/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs deleted file mode 100644 index a8c7f1fd4..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs +++ /dev/null @@ -1,317 +0,0 @@ -using Microsoft.AspNetCore.Http; -using Serval.Translation.V1; - -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMachineBuilderExtensions -{ - public static IMachineBuilder AddServiceOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddServiceOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddClearMLNmtEngineOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddClearMLNmtEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddThotSmtModel( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder.AddThotSmtModel(); - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder.AddThotSmtModel(); - } - - public static IMachineBuilder AddTransferEngine(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddUnigramTruecaser(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddClearMLService(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); - - // workaround register satisfying the interface and as a hosted service. - builder.Services.AddSingleton(); - builder.Services.AddHostedService(p => p.GetRequiredService()); - - builder.Services - .AddHttpClient() - .AddTransientHttpErrorPolicy( - b => b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) - ); - - return builder; - } - - public static IMachineBuilder AddMongoBackgroundJobClient( - this IMachineBuilder builder, - string? connectionString = null - ) - { - builder.Services.AddHangfire( - c => - c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) - .UseSimpleAssemblyNameTypeSerializer() - .UseRecommendedSerializerSettings() - .UseMongoStorage( - connectionString ?? builder.Configuration.GetConnectionString("Hangfire"), - new MongoStorageOptions - { - MigrationOptions = new MongoMigrationOptions - { - MigrationStrategy = new MigrateMongoMigrationStrategy(), - BackupStrategy = new CollectionMongoBackupStrategy() - }, - CheckConnection = true, - CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, - } - ) - ); - builder.Services.AddHealthChecks().AddCheck(name: "Hangfire"); - return builder; - } - - public static IMachineBuilder AddBackgroundJobServer( - this IMachineBuilder builder, - IEnumerable? engineTypes = null - ) - { - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; - var queues = new List(); - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - queues.Add("smt_transfer"); - break; - case TranslationEngineType.Nmt: - builder.AddClearMLService(); - queues.Add("nmt"); - break; - } - } - - builder.Services.AddHangfireServer(o => - { - o.Queues = queues.ToArray(); - }); - return builder; - } - - public static IMachineBuilder AddMemoryDataAccess(this IMachineBuilder builder) - { - builder.Services.AddMemoryDataAccess(o => - { - o.AddRepository(); - o.AddRepository(); - o.AddRepository(); - }); - - return builder; - } - - public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, string? connectionString = null) - { - connectionString ??= builder.Configuration.GetConnectionString("Mongo"); - builder.Services.AddMongoDataAccess( - connectionString, - "SIL.Machine.AspNetCore.Models", - o => - { - o.AddRepository( - "translation_engines", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.EngineId) - ) - ) - ); - o.AddRepository( - "locks", - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("writerLock._id")) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("readerLocks._id")) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("writerQueue._id")) - ); - } - ); - o.AddRepository( - "train_segment_pairs", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) - ) - ) - ); - } - ); - builder.Services.AddHealthChecks().AddMongoDb(connectionString, name: "Mongo"); - - return builder; - } - - public static IMachineBuilder AddServalPlatformService( - this IMachineBuilder builder, - string? connectionString = null - ) - { - builder.Services.AddScoped(); - builder.Services - .AddGrpcClient(o => - { - o.Address = new Uri(connectionString ?? builder.Configuration.GetConnectionString("Serval")); - }) - .ConfigureChannel(o => - { - o.MaxRetryAttempts = null; - o.ServiceConfig = new ServiceConfig - { - MethodConfigs = - { - new MethodConfig - { - Names = { MethodName.Default }, - RetryPolicy = new RetryPolicy - { - MaxAttempts = 10, - InitialBackoff = TimeSpan.FromSeconds(1), - MaxBackoff = TimeSpan.FromSeconds(5), - BackoffMultiplier = 1.5, - RetryableStatusCodes = { StatusCode.Unavailable } - } - }, - new MethodConfig - { - Names = - { - new MethodName - { - Service = "serval.translation.v1.TranslationPlatformApi", - Method = "UpdateBuildStatus" - } - } - }, - } - }; - }); - - return builder; - } - - public static IMachineBuilder AddServalTranslationEngineService( - this IMachineBuilder builder, - string? connectionString = null, - IEnumerable? engineTypes = null - ) - { - builder.Services.AddGrpc(options => options.Interceptors.Add()); - builder.AddServalPlatformService(connectionString ?? builder.Configuration.GetConnectionString("Serval")); - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.Services.AddSingleton(); - builder.Services.AddHostedService(); - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - builder.Services.AddScoped(); - break; - case TranslationEngineType.Nmt: - builder.AddClearMLService(); - builder.Services.AddScoped(); - break; - } - } - builder.Services.AddGrpcHealthChecks(); - - return builder; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs b/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs deleted file mode 100644 index 739a72924..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs +++ /dev/null @@ -1,317 +0,0 @@ -using Microsoft.AspNetCore.Http; -using Serval.Translation.V1; - -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMachineBuilderExtensions -{ - public static IMachineBuilder AddServiceOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddServiceOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddClearMLNmtEngineOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddClearMLNmtEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddThotSmtModel( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder.AddThotSmtModel(); - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder.AddThotSmtModel(); - } - - public static IMachineBuilder AddTransferEngine(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddUnigramTruecaser(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddClearMLService(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); - - // workaround register satisfying the interface and as a hosted service. - builder.Services.AddSingleton(); - builder.Services.AddHostedService(p => p.GetRequiredService()); - - builder.Services - .AddHttpClient() - .AddTransientHttpErrorPolicy( - b => b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) - ); - - return builder; - } - - public static IMachineBuilder AddMongoBackgroundJobClient( - this IMachineBuilder builder, - string? connectionString = null - ) - { - builder.Services.AddHangfire( - c => - c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) - .UseSimpleAssemblyNameTypeSerializer() - .UseRecommendedSerializerSettings() - .UseMongoStorage( - connectionString ?? builder.Configuration.GetConnectionString("Hangfire"), - new MongoStorageOptions - { - MigrationOptions = new MongoMigrationOptions - { - MigrationStrategy = new MigrateMongoMigrationStrategy(), - BackupStrategy = new CollectionMongoBackupStrategy() - }, - CheckConnection = true, - CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, - } - ) - ); - builder.Services.AddHealthChecks().AddCheck(name: "Hangfire"); - return builder; - } - - public static IMachineBuilder AddBackgroundJobServer( - this IMachineBuilder builder, - IEnumerable? engineTypes = null - ) - { - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; - var queues = new List(); - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - queues.Add("smt_transfer"); - break; - case TranslationEngineType.Nmt: - builder.AddClearMLService(); - queues.Add("nmt"); - break; - } - } - - builder.Services.AddHangfireServer(o => - { - o.Queues = queues.ToArray(); - }); - return builder; - } - - public static IMachineBuilder AddMemoryDataAccess(this IMachineBuilder builder) - { - builder.Services.AddMemoryDataAccess(o => - { - o.AddRepository(); - o.AddRepository(); - o.AddRepository(); - }); - - return builder; - } - - public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, string? connectionString = null) - { - connectionString ??= builder.Configuration.GetConnectionString("Mongo"); - builder.Services.AddMongoDataAccess( - connectionString, - "SIL.Machine.AspNetCore.Models", - o => - { - o.AddRepository( - "translation_engines", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.EngineId) - ) - ) - ); - o.AddRepository( - "locks", - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("writerLock._id")) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("readerLocks._id")) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("writerQueue._id")) - ); - } - ); - o.AddRepository( - "train_segment_pairs", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) - ) - ) - ); - } - ); - builder.Services.AddHealthChecks().AddMongoDb(connectionString, name: "Mongo"); - - return builder; - } - - public static IMachineBuilder AddServalPlatformService( - this IMachineBuilder builder, - string? connectionString = null - ) - { - builder.Services.AddScoped(); - builder.Services - .AddGrpcClient(o => - { - o.Address = new Uri(connectionString ?? builder.Configuration.GetConnectionString("Serval")); - }) - .ConfigureChannel(o => - { - o.MaxRetryAttempts = null; - o.ServiceConfig = new ServiceConfig - { - MethodConfigs = - { - new MethodConfig - { - Names = { MethodName.Default }, - RetryPolicy = new Grpc.Net.Client.Configuration.RetryPolicy - { - MaxAttempts = 10, - InitialBackoff = TimeSpan.FromSeconds(1), - MaxBackoff = TimeSpan.FromSeconds(5), - BackoffMultiplier = 1.5, - RetryableStatusCodes = { StatusCode.Unavailable } - } - }, - new MethodConfig - { - Names = - { - new MethodName - { - Service = "serval.translation.v1.TranslationPlatformApi", - Method = "UpdateBuildStatus" - } - } - }, - } - }; - }); - - return builder; - } - - public static IMachineBuilder AddServalTranslationEngineService( - this IMachineBuilder builder, - string? connectionString = null, - IEnumerable? engineTypes = null - ) - { - builder.Services.AddGrpc(options => options.Interceptors.Add()); - builder.AddServalPlatformService(connectionString ?? builder.Configuration.GetConnectionString("Serval")); - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.Services.AddSingleton(); - builder.Services.AddHostedService(); - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - builder.Services.AddScoped(); - break; - case TranslationEngineType.Nmt: - builder.AddClearMLService(); - builder.Services.AddScoped(); - break; - } - } - builder.Services.AddGrpcHealthChecks(); - - return builder; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs deleted file mode 100644 index 9e7eed761..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs +++ /dev/null @@ -1,343 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await sourceTrainWriter.BaseStream.DisposeAsync(); - - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs deleted file mode 100644 index e41df89d3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs +++ /dev/null @@ -1,342 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await targetTrainWriter.BaseStream.DisposeAsync(); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs deleted file mode 100644 index e41df89d3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs +++ /dev/null @@ -1,342 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await targetTrainWriter.BaseStream.DisposeAsync(); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs deleted file mode 100644 index e41df89d3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs +++ /dev/null @@ -1,342 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await targetTrainWriter.BaseStream.DisposeAsync(); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs deleted file mode 100644 index e41df89d3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs +++ /dev/null @@ -1,342 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await targetTrainWriter.BaseStream.DisposeAsync(); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs deleted file mode 100644 index bf3981d14..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs +++ /dev/null @@ -1,342 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await targetTrainWriter.BaseStream.DisposeAsync(); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs deleted file mode 100644 index a26cda901..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs deleted file mode 100644 index 42eb064e1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs +++ /dev/null @@ -1,51 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineService : TranslationEngineServiceBase -{ - private readonly IClearMLService _clearMLService; - - public ClearMLNmtEngineService( - IBackgroundJobClient jobClient, - IPlatformService platformService, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IRepository engines, - IClearMLService clearMLService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _clearMLService = clearMLService; - } - - public override TranslationEngineType Type => TranslationEngineType.Nmt; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - await _clearMLService.CreateProjectAsync(engineId, engineName, cancellationToken: CancellationToken.None); - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - string? projectId = await _clearMLService.GetProjectIdAsync(engineId, CancellationToken.None); - if (projectId is not null) - await _clearMLService.DeleteProjectAsync(projectId, CancellationToken.None); - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs deleted file mode 100644 index 42eb064e1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs +++ /dev/null @@ -1,51 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineService : TranslationEngineServiceBase -{ - private readonly IClearMLService _clearMLService; - - public ClearMLNmtEngineService( - IBackgroundJobClient jobClient, - IPlatformService platformService, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IRepository engines, - IClearMLService clearMLService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _clearMLService = clearMLService; - } - - public override TranslationEngineType Type => TranslationEngineType.Nmt; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - await _clearMLService.CreateProjectAsync(engineId, engineName, cancellationToken: CancellationToken.None); - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - string? projectId = await _clearMLService.GetProjectIdAsync(engineId, CancellationToken.None); - if (projectId is not null) - await _clearMLService.DeleteProjectAsync(projectId, CancellationToken.None); - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs deleted file mode 100644 index 42eb064e1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs +++ /dev/null @@ -1,51 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineService : TranslationEngineServiceBase -{ - private readonly IClearMLService _clearMLService; - - public ClearMLNmtEngineService( - IBackgroundJobClient jobClient, - IPlatformService platformService, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IRepository engines, - IClearMLService clearMLService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _clearMLService = clearMLService; - } - - public override TranslationEngineType Type => TranslationEngineType.Nmt; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - await _clearMLService.CreateProjectAsync(engineId, engineName, cancellationToken: CancellationToken.None); - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - string? projectId = await _clearMLService.GetProjectIdAsync(engineId, CancellationToken.None); - if (projectId is not null) - await _clearMLService.DeleteProjectAsync(projectId, CancellationToken.None); - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs deleted file mode 100644 index 56bbc9829..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs +++ /dev/null @@ -1,95 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - return Task.FromResult( - new BufferedStream(new S3WriteStream(_client, objectId, _bucketName), 1024 * 1024 * 100) - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs deleted file mode 100644 index 56bbc9829..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs +++ /dev/null @@ -1,95 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - return Task.FromResult( - new BufferedStream(new S3WriteStream(_client, objectId, _bucketName), 1024 * 1024 * 100) - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs deleted file mode 100644 index aaf4a3f52..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs +++ /dev/null @@ -1,97 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - 1024 * 1024 * 5 - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs deleted file mode 100644 index 66204bcaa..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs +++ /dev/null @@ -1,97 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - 1024 * 1024 * 5)); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs deleted file mode 100644 index 413860de7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs +++ /dev/null @@ -1,98 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - 1024 * 1024 * 5 - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs deleted file mode 100644 index 413860de7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs +++ /dev/null @@ -1,98 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - 1024 * 1024 * 5 - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs deleted file mode 100644 index 1eeacbc76..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs +++ /dev/null @@ -1,98 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs deleted file mode 100644 index 1eeacbc76..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs +++ /dev/null @@ -1,98 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs deleted file mode 100644 index 2ae672c3c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs deleted file mode 100644 index 0619ee785..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs deleted file mode 100644 index b5eb883cb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - S3WriteStream.FiveMB, - _loggerFactory - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs deleted file mode 100644 index 9058da211..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId, _loggerFactory), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs deleted file mode 100644 index 9058da211..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId, _loggerFactory), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs deleted file mode 100644 index ff8563a42..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId, _loggerFactory), - S3WriteStream.MaxPartSize - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs deleted file mode 100644 index b2d5808f0..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs +++ /dev/null @@ -1,76 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _bucketName; - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _length = 0; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs deleted file mode 100644 index e27f3416a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs +++ /dev/null @@ -1,76 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services;S3Wri - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _bucketName; - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _length = 0; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs deleted file mode 100644 index b2d5808f0..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs +++ /dev/null @@ -1,76 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _bucketName; - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _length = 0; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs deleted file mode 100644 index a4284a6fe..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs +++ /dev/null @@ -1,77 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _bucketName; - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _length = 0; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs deleted file mode 100644 index effbfca90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs +++ /dev/null @@ -1,79 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs deleted file mode 100644 index 1fcc576e6..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs +++ /dev/null @@ -1,78 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs deleted file mode 100644 index 213ddd80a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs +++ /dev/null @@ -1,78 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs deleted file mode 100644 index 24f757833..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs +++ /dev/null @@ -1,87 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - FilePosition = _length - }; - _length += count; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs deleted file mode 100644 index 04454309b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs +++ /dev/null @@ -1,75 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - FilePosition = _length - }; - _length += count; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs deleted file mode 100644 index 46a493f7f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - FilePosition = _length - }; - _length += count; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs deleted file mode 100644 index 46a493f7f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - FilePosition = _length - }; - _length += count; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs deleted file mode 100644 index a5cdde5c5..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs +++ /dev/null @@ -1,106 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber - }; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs deleted file mode 100644 index 16f137a5a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber - }; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs deleted file mode 100644 index 7d72d7d4d..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - Stream = ms - }; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs deleted file mode 100644 index 21f06731c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs deleted file mode 100644 index 21f06731c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs deleted file mode 100644 index c551af600..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs +++ /dev/null @@ -1,109 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - UploadPartResponse response = await _client.UploadPartAsync(request); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs deleted file mode 100644 index 859618376..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs +++ /dev/null @@ -1,110 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - UploadPartResponse response = await _client.UploadPartAsync(request); - if(response.HttpStatusCode != HttpStatusCode.OK) throw new HttpRequestException($"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}") - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs deleted file mode 100644 index 38f8cf6d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs +++ /dev/null @@ -1,113 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs deleted file mode 100644 index 4e5370496..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs +++ /dev/null @@ -1,116 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - uploadRequest.StreamTransferProgress += new EventHandler( - UploadPartProgressEventCallback - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs deleted file mode 100644 index e89e96826..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs +++ /dev/null @@ -1,114 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler((_, e) => { }); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs deleted file mode 100644 index 7be03bf75..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs +++ /dev/null @@ -1,113 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler((_, e) => { }); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs deleted file mode 100644 index 785cf5858..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs +++ /dev/null @@ -1,114 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler((_, e) => { }); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs deleted file mode 100644 index 64918822a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs +++ /dev/null @@ -1,115 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = (new LoggerFactory()).CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler((_, e) => { }); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs deleted file mode 100644 index 3a05d9ab0..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs +++ /dev/null @@ -1,115 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler((_, e) => { }); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs deleted file mode 100644 index 2d1c7f47a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs deleted file mode 100644 index 2d1c7f47a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs deleted file mode 100644 index 7c5630f4b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs deleted file mode 100644 index 7c5630f4b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs deleted file mode 100644 index 52ecc2ef3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs deleted file mode 100644 index 52ecc2ef3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs deleted file mode 100644 index cb6f6edb5..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs deleted file mode 100644 index d6bc7c0aa..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs deleted file mode 100644 index 81005c673..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs deleted file mode 100644 index bff5aa1d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs deleted file mode 100644 index ac7aa225c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs +++ /dev/null @@ -1,124 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs deleted file mode 100644 index 5f5b59245..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs deleted file mode 100644 index 530e928ca..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs +++ /dev/null @@ -1,124 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs deleted file mode 100644 index 530e928ca..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs +++ /dev/null @@ -1,124 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs deleted file mode 100644 index 7e13f6a62..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs deleted file mode 100644 index 104dc7116..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs deleted file mode 100644 index 104dc7116..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs deleted file mode 100644 index a97937131..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs +++ /dev/null @@ -1,126 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs deleted file mode 100644 index a97937131..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs +++ /dev/null @@ -1,126 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs deleted file mode 100644 index a97937131..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs +++ /dev/null @@ -1,126 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs deleted file mode 100644 index 1024e4aec..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs +++ /dev/null @@ -1,127 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) { } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs deleted file mode 100644 index 4ef04e880..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs +++ /dev/null @@ -1,151 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs deleted file mode 100644 index e4c5eb276..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs +++ /dev/null @@ -1,151 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client.CompleteMultipartUpload(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs deleted file mode 100644 index 9a03bb2e4..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs deleted file mode 100644 index 584a0e09c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs +++ /dev/null @@ -1,153 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs deleted file mode 100644 index 9b4f9cf96..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public static final int fiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs deleted file mode 100644 index d7b0b6982..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public final static int fiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs deleted file mode 100644 index a47284fb5..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public static const int fiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs deleted file mode 100644 index 50a58fafe..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int fiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs deleted file mode 100644 index c13ee9bc2..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FIVE_MB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs deleted file mode 100644 index c13ee9bc2..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FIVE_MB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs deleted file mode 100644 index 20cee4645..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMb = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs deleted file mode 100644 index 262f9a6c3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMbB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs deleted file mode 100644 index 9ea77dfee..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs deleted file mode 100644 index 9ea77dfee..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs deleted file mode 100644 index 6d25881bb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs deleted file mode 100644 index 6d25881bb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs deleted file mode 100644 index 95dd7899d..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs +++ /dev/null @@ -1,161 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs deleted file mode 100644 index 001301acb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs +++ /dev/null @@ -1,161 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs deleted file mode 100644 index 001301acb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs +++ /dev/null @@ -1,161 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs deleted file mode 100644 index 7fc4c6f7a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs +++ /dev/null @@ -1,162 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs deleted file mode 100644 index afd96e3f9..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs deleted file mode 100644 index afd96e3f9..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs deleted file mode 100644 index 339ca3a26..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs deleted file mode 100644 index 339ca3a26..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs deleted file mode 100644 index fc2173053..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int MaxPartSize = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = MaxPartSize - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs deleted file mode 100644 index fc2173053..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int MaxPartSize = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = MaxPartSize - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs deleted file mode 100644 index fc2173053..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int MaxPartSize = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = MaxPartSize - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs deleted file mode 100644 index 526cf95fd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs +++ /dev/null @@ -1,92 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - - public SharedFileService(IOptions? options = null) - { - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs deleted file mode 100644 index 526cf95fd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs +++ /dev/null @@ -1,92 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - - public SharedFileService(IOptions? options = null) - { - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs deleted file mode 100644 index a7a778353..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs +++ /dev/null @@ -1,92 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - - public SharedFileService(IOptions? options = null) - { - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - }} - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs deleted file mode 100644 index 526cf95fd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs +++ /dev/null @@ -1,92 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - - public SharedFileService(IOptions? options = null) - { - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs deleted file mode 100644 index c7eb4f847..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs +++ /dev/null @@ -1,95 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) - { - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs deleted file mode 100644 index 53777aeaa..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs +++ /dev/null @@ -1,97 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs deleted file mode 100644 index c9f4aa6b1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs +++ /dev/null @@ -1,98 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs deleted file mode 100644 index c9f4aa6b1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs +++ /dev/null @@ -1,98 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs deleted file mode 100644 index 5804584c1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs +++ /dev/null @@ -1,98 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs deleted file mode 100644 index 5804584c1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs +++ /dev/null @@ -1,98 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs deleted file mode 100644 index cef73bbeb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs +++ /dev/null @@ -1,96 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs deleted file mode 100644 index cef73bbeb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs +++ /dev/null @@ -1,96 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs deleted file mode 100644 index e7619bc63..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs +++ /dev/null @@ -1,181 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs deleted file mode 100644 index 804a21c2e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs +++ /dev/null @@ -1,183 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - if (state.CurrentBuildRevision == -1) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs deleted file mode 100644 index 804a21c2e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs +++ /dev/null @@ -1,183 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - if (state.CurrentBuildRevision == -1) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs deleted file mode 100644 index 804a21c2e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs +++ /dev/null @@ -1,183 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - if (state.CurrentBuildRevision == -1) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs deleted file mode 100644 index 68d6dc96e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs +++ /dev/null @@ -1,183 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - if (state.CurrentBuildRevision == -1) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs deleted file mode 100644 index 38b27e746..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if (await Engines.GetAsync(e => e.Id == engineId && e.BuildId)) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs deleted file mode 100644 index 4c6134aa9..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - ( - await Engines.GetAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs deleted file mode 100644 index cf92eef1a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - ( - await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs deleted file mode 100644 index cf92eef1a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - ( - await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs deleted file mode 100644 index 4672cf290..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - !( - await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs deleted file mode 100644 index 4672cf290..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - !( - await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs deleted file mode 100644 index f75ec89f0..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs +++ /dev/null @@ -1,190 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs deleted file mode 100644 index f75ec89f0..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs +++ /dev/null @@ -1,190 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs deleted file mode 100644 index 22727b2d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs +++ /dev/null @@ -1,189 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs deleted file mode 100644 index 22727b2d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs +++ /dev/null @@ -1,189 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs deleted file mode 100644 index 22727b2d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs +++ /dev/null @@ -1,189 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs deleted file mode 100644 index 22727b2d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs +++ /dev/null @@ -1,189 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs deleted file mode 100644 index f5d00996f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs +++ /dev/null @@ -1,190 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - if ( - !( - - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs deleted file mode 100644 index dd3aa1037..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs +++ /dev/null @@ -1,192 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach(var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if ( - !( - - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs deleted file mode 100644 index b773a87c2..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!(engines).Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs deleted file mode 100644 index 46ad52609..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs +++ /dev/null @@ -1,189 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Console.WriteLine("|||||||||||||||||||"); - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs deleted file mode 100644 index 9991f0c87..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs +++ /dev/null @@ -1,191 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Console.WriteLine("|||||||||||||||||||"); - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - Console.WriteLine("|||||||||||||||||||"); - - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs deleted file mode 100644 index 3574fa79f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs +++ /dev/null @@ -1,190 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Console.WriteLine("|||||||||||||||||||"); - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - Console.WriteLine("|||||||||||||||||||"); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs deleted file mode 100644 index 47ce83b1f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs +++ /dev/null @@ -1,190 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Console.WriteLine("|||||||||||||||||||"); - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - Console.WriteLine("|||||||||||||||||||"); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs deleted file mode 100644 index 234a20433..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Console.WriteLine("|||||||||||||||||||"); - IEnumerable engines = await Engines.GetAllAsync(e => e.Id == engineId); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - Console.WriteLine("|||||||||||||||||||"); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs deleted file mode 100644 index 92d3d3d1e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs +++ /dev/null @@ -1,185 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs deleted file mode 100644 index 26186680c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs deleted file mode 100644 index eaa649a10..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs +++ /dev/null @@ -1,185 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs deleted file mode 100644 index 76069459f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildRevision == 0 || engine.BuildState != BuildState.None || engine.Revision == 0) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs deleted file mode 100644 index 76069459f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildRevision == 0 || engine.BuildState != BuildState.None || engine.Revision == 0) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs deleted file mode 100644 index 76069459f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildRevision == 0 || engine.BuildState != BuildState.None || engine.Revision == 0) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs deleted file mode 100644 index b73ba6ead..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.Revision == 0) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs deleted file mode 100644 index b73ba6ead..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.Revision == 0) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs deleted file mode 100644 index ebb403ba9..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.Revision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs deleted file mode 100644 index 296b73779..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs deleted file mode 100644 index 296b73779..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs deleted file mode 100644 index 296b73779..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs deleted file mode 100644 index 296b73779..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs deleted file mode 100644 index 3ab3f9d3f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs deleted file mode 100644 index e4be91c25..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs deleted file mode 100644 index 91a738627..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs +++ /dev/null @@ -1,175 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs deleted file mode 100644 index e4eba1d38..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs +++ /dev/null @@ -1,174 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs deleted file mode 100644 index b4dde802b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs +++ /dev/null @@ -1,175 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs deleted file mode 100644 index 0eae7ff8b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs +++ /dev/null @@ -1,174 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs deleted file mode 100644 index 6bc79f5bd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs deleted file mode 100644 index 6bc79f5bd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs deleted file mode 100644 index 8538f7d87..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await G(engineetBuiltEngineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs deleted file mode 100644 index 6bc79f5bd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs deleted file mode 100644 index 67a1023da..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngine(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs deleted file mode 100644 index 2cfccdc7e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs deleted file mode 100644 index 2cfccdc7e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs deleted file mode 100644 index 0f3d710af..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs +++ /dev/null @@ -1,212 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs deleted file mode 100644 index 41ff71d4b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs +++ /dev/null @@ -1,219 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs deleted file mode 100644 index d7acd2bfd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs +++ /dev/null @@ -1,219 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs deleted file mode 100644 index 2d7e5e928..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs +++ /dev/null @@ -1,227 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs deleted file mode 100644 index 2d7e5e928..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs +++ /dev/null @@ -1,227 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs deleted file mode 100644 index 2d7e5e928..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs +++ /dev/null @@ -1,227 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs deleted file mode 100644 index 2d7e5e928..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs +++ /dev/null @@ -1,227 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs deleted file mode 100644 index 8c2e0cbc3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs +++ /dev/null @@ -1,230 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt( - string engineId, - CancellationToken cancellationToken - ) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs deleted file mode 100644 index f310e85f3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs +++ /dev/null @@ -1,227 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs deleted file mode 100644 index dc702f8de..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs deleted file mode 100644 index dc702f8de..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs deleted file mode 100644 index d6511bf11..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs deleted file mode 100644 index dc702f8de..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs deleted file mode 100644 index dc702f8de..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs deleted file mode 100644 index 21868db71..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs +++ /dev/null @@ -1,47 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.S3.Transfer; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs deleted file mode 100644 index 0b07fae03..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs +++ /dev/null @@ -1,47 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.Runtime; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs deleted file mode 100644 index a808510e6..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs +++ /dev/null @@ -1,49 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.Runtime; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -using Nito.AsyncEx.Synchronous; - -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs deleted file mode 100644 index 981c59ef5..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs +++ /dev/null @@ -1,48 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.Runtime; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -using Nito.AsyncEx.Synchronous; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs deleted file mode 100644 index 8544d1e96..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs +++ /dev/null @@ -1,48 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.Runtime; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -global using Nito.AsyncEx.Synchronous; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs deleted file mode 100644 index 8544d1e96..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs +++ /dev/null @@ -1,48 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.Runtime; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -global using Nito.AsyncEx.Synchronous; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs b/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs deleted file mode 100644 index f9d01a7d6..000000000 --- a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs +++ /dev/null @@ -1,192 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class ClearMLNmtEngineServiceTests -{ - [Test] - public async Task CancelBuildAsync() - { - using var env = new TestEnvironment(); - env.ClearMLService - .CreateTaskAsync( - Arg.Any(), - "project1", - "engine1", - "es", - "en", - "memory:///", - Arg.Any() - ) - .Returns(Task.FromResult("task1")); - var task = new ClearMLTask - { - Id = "task1", - Project = new ClearMLProject { Id = "project1" }, - Status = ClearMLTaskStatus.InProgress - }; - bool first = true; - env.ClearMLService - .GetTaskByNameAsync(Arg.Any(), Arg.Any()) - .Returns(x => - { - if (first) - { - first = false; - return Task.FromResult(null); - } - return Task.FromResult(task); - }); - env.ClearMLService - .GetTaskByIdAsync("task1", Arg.Any()) - .Returns(Task.FromResult(task)); - await env.Service.StartBuildAsync("engine1", "build1", Array.Empty()); - await env.WaitForBuildToStartAsync(); - TranslationEngine engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildState, Is.EqualTo(BuildState.Active)); - await env.Service.CancelBuildAsync("engine1"); - await env.WaitForBuildToFinishAsync(); - engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildState, Is.EqualTo(BuildState.None)); - await env.ClearMLService.Received().StopTaskAsync("task1", Arg.Any()); - } - - private class TestEnvironment : DisposableBase - { - private readonly MemoryStorage _memoryStorage; - private readonly BackgroundJobClient _jobClient; - private BackgroundJobServer _jobServer; - private readonly IDistributedReaderWriterLockFactory _lockFactory; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - - public TestEnvironment() - { - Engines = new MemoryRepository(); - Engines.Add( - new TranslationEngine - { - Id = "engine1", - EngineId = "engine1", - SourceLanguage = "es", - TargetLanguage = "en" - } - ); - EngineOptions = new SmtTransferEngineOptions(); - _memoryStorage = new MemoryStorage(); - _jobClient = new BackgroundJobClient(_memoryStorage); - PlatformService = Substitute.For(); - ClearMLService = Substitute.For(); - ClearMLService - .GetProjectIdAsync(Arg.Any(), Arg.Any()) - .Returns(Task.FromResult("project1")); - _lockFactory = new DistributedReaderWriterLockFactory( - new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), - new MemoryRepository(), - new ObjectIdGenerator() - ); - _sharedFileService = new SharedFileService(); - _options = Substitute.For>(); - _options.CurrentValue.Returns( - new ClearMLNmtEngineOptions { BuildPollingTimeout = TimeSpan.FromMilliseconds(50) } - ); - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - public ClearMLNmtEngineService Service { get; private set; } - public MemoryRepository Engines { get; } - public SmtTransferEngineOptions EngineOptions { get; } - public IPlatformService PlatformService { get; } - public IClearMLService ClearMLService { get; } - - public void StopServer() - { - _jobServer.Dispose(); - } - - public void StartServer() - { - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - private BackgroundJobServer CreateJobServer() - { - var jobServerOptions = new BackgroundJobServerOptions - { - Activator = new EnvActivator(this), - Queues = new[] { "nmt" }, - CancellationCheckInterval = TimeSpan.FromMilliseconds(100), - }; - return new BackgroundJobServer(jobServerOptions, _memoryStorage); - } - - private ClearMLNmtEngineService CreateService() - { - return new ClearMLNmtEngineService( - _jobClient, - PlatformService, - _lockFactory, - new MemoryDataAccessContext(), - Engines, - ClearMLService - ); - } - - public Task WaitForBuildToFinishAsync() - { - return WaitForBuildState(e => e.BuildState is BuildState.None); - } - - public Task WaitForBuildToStartAsync() - { - return WaitForBuildState(e => e.BuildState is BuildState.Active); - } - - private async Task WaitForBuildState(Func predicate) - { - using ISubscription subscription = await Engines.SubscribeAsync( - e => e.EngineId == "engine1" - ); - while (true) - { - TranslationEngine? build = subscription.Change.Entity; - if (build is not null && predicate(build)) - break; - await subscription.WaitForChangeAsync(); - } - } - - protected override void DisposeManagedResources() - { - _jobServer.Dispose(); - } - - private class EnvActivator : JobActivator - { - private readonly TestEnvironment _env; - - public EnvActivator(TestEnvironment env) - { - _env = env; - } - - public override object ActivateJob(Type jobType) - { - if (jobType == typeof(ClearMLNmtEngineBuildJob)) - { - return new ClearMLNmtEngineBuildJob( - _env.PlatformService, - _env.Engines, - Substitute.For>(), - _env.ClearMLService, - _env._sharedFileService, - _env._options, - Substitute.For() - ); - } - return base.ActivateJob(jobType); - } - } - } -} diff --git a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs b/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs deleted file mode 100644 index 614dabc52..000000000 --- a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs +++ /dev/null @@ -1,192 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class ClearMLNmtEngineServiceTests -{ - [Test] - public async Task CancelBuildAsync() - { - using var env = new TestEnvironment(); - env.ClearMLService - .CreateTaskAsync( - Arg.Any(), - "project1", - "engine1", - "es", - "en", - "memory:///", - Arg.Any() - ) - .Returns(Task.FromResult("task1")); - var task = new ClearMLTask - { - Id = "task1", - Project = new ClearMLProject { Id = "project1" }, - Status = ClearMLTaskStatus.InProgress - }; - bool first = true; - env.ClearMLService - .GetTaskByNameAsync(Arg.Any(), Arg.Any()) - .Returns(x => - { - if (first) - { - first = false; - return Task.FromResult(null); - } - return Task.FromResult(task); - }); - env.ClearMLService - .GetTaskByIdAsync("task1", Arg.Any()) - .Returns(Task.FromResult(task)); - await env.Service.StartBuildAsync("engine1", "build1", Array.Empty()); - await env.WaitForBuildToStartAsync(); - TranslationEngine engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildState, Is.EqualTo(BuildState.Active)); - await env.Service.CancelBuildAsync("engine1"); - await env.WaitForBuildToFinishAsync(); - engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildState, Is.EqualTo(BuildState.None)); - await env.ClearMLService.Received().StopTaskAsync("task1", Arg.Any()); - } - - private class TestEnvironment : DisposableBase - { - private readonly MemoryStorage _memoryStorage; - private readonly BackgroundJobClient _jobClient; - private BackgroundJobServer _jobServer; - private readonly IDistributedReaderWriterLockFactory _lockFactory; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - - public TestEnvironment() - { - Engines = new MemoryRepository(); - Engines.Add( - new TranslationEngine - { - Id = "engine1", - EngineId = "engine1", - SourceLanguage = "es", - TargetLanguage = "en" - } - ); - EngineOptions = new SmtTransferEngineOptions(); - _memoryStorage = new MemoryStorage(); - _jobClient = new BackgroundJobClient(_memoryStorage); - PlatformService = Substitute.For(); - ClearMLService = Substitute.For(); - ClearMLService - .GetProjectIdAsync(Arg.Any(), Arg.Any()) - .Returns(Task.FromResult("project1")); - _lockFactory = new DistributedReaderWriterLockFactory( - new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), - new MemoryRepository(), - new ObjectIdGenerator() - ); - _sharedFileService = new SharedFileService(Substitute.For()); - _options = Substitute.For>(); - _options.CurrentValue.Returns( - new ClearMLNmtEngineOptions { BuildPollingTimeout = TimeSpan.FromMilliseconds(50) } - ); - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - public ClearMLNmtEngineService Service { get; private set; } - public MemoryRepository Engines { get; } - public SmtTransferEngineOptions EngineOptions { get; } - public IPlatformService PlatformService { get; } - public IClearMLService ClearMLService { get; } - - public void StopServer() - { - _jobServer.Dispose(); - } - - public void StartServer() - { - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - private BackgroundJobServer CreateJobServer() - { - var jobServerOptions = new BackgroundJobServerOptions - { - Activator = new EnvActivator(this), - Queues = new[] { "nmt" }, - CancellationCheckInterval = TimeSpan.FromMilliseconds(100), - }; - return new BackgroundJobServer(jobServerOptions, _memoryStorage); - } - - private ClearMLNmtEngineService CreateService() - { - return new ClearMLNmtEngineService( - _jobClient, - PlatformService, - _lockFactory, - new MemoryDataAccessContext(), - Engines, - ClearMLService - ); - } - - public Task WaitForBuildToFinishAsync() - { - return WaitForBuildState(e => e.BuildState is BuildState.None); - } - - public Task WaitForBuildToStartAsync() - { - return WaitForBuildState(e => e.BuildState is BuildState.Active); - } - - private async Task WaitForBuildState(Func predicate) - { - using ISubscription subscription = await Engines.SubscribeAsync( - e => e.EngineId == "engine1" - ); - while (true) - { - TranslationEngine? build = subscription.Change.Entity; - if (build is not null && predicate(build)) - break; - await subscription.WaitForChangeAsync(); - } - } - - protected override void DisposeManagedResources() - { - _jobServer.Dispose(); - } - - private class EnvActivator : JobActivator - { - private readonly TestEnvironment _env; - - public EnvActivator(TestEnvironment env) - { - _env = env; - } - - public override object ActivateJob(Type jobType) - { - if (jobType == typeof(ClearMLNmtEngineBuildJob)) - { - return new ClearMLNmtEngineBuildJob( - _env.PlatformService, - _env.Engines, - Substitute.For>(), - _env.ClearMLService, - _env._sharedFileService, - _env._options, - Substitute.For() - ); - } - return base.ActivateJob(jobType); - } - } - } -}