From 3fab687398ad4d5ab623675c844c45b043941ab0 Mon Sep 17 00:00:00 2001 From: Enkidu93 Date: Mon, 11 Sep 2023 13:05:35 -0400 Subject: [PATCH] Revert "Empty commit to trigger workflow --ECL" This reverts commit c9384c86e4886efa63fa416e76571f0c88b7f0e6. --- .history/.gitignore_20230911115750 | 51 --- .history/.gitignore_20230911115828 | 52 --- ...MachineBuilderExtensions_20230907084316.cs | 317 ---------------- ...MachineBuilderExtensions_20230907133310.cs | 317 ---------------- ...ClearMLNmtEngineBuildJob_20230907084316.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907095831.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907135725.cs | 343 ------------------ ...ClearMLNmtEngineBuildJob_20230907135737.cs | 342 ----------------- ...ClearMLNmtEngineBuildJob_20230907135738.cs | 342 ----------------- ...ClearMLNmtEngineBuildJob_20230907135739.cs | 342 ----------------- ...ClearMLNmtEngineBuildJob_20230907140302.cs | 342 ----------------- ...ClearMLNmtEngineBuildJob_20230907140611.cs | 342 ----------------- ...ClearMLNmtEngineBuildJob_20230907140709.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907140719.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907140720.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907141110.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907141112.cs | 340 ----------------- ...ClearMLNmtEngineBuildJob_20230907141905.cs | 340 ----------------- .../ClearMLNmtEngineService_20230906141540.cs | 51 --- .../ClearMLNmtEngineService_20230911085826.cs | 51 --- .../ClearMLNmtEngineService_20230911085829.cs | 51 --- .../Services/S3FileStorage_20230906141540.cs | 95 ----- .../Services/S3FileStorage_20230907104119.cs | 95 ----- .../Services/S3FileStorage_20230907130153.cs | 97 ----- .../Services/S3FileStorage_20230907130157.cs | 97 ----- .../Services/S3FileStorage_20230907132651.cs | 98 ----- .../Services/S3FileStorage_20230907132652.cs | 98 ----- .../Services/S3FileStorage_20230907171652.cs | 98 ----- .../Services/S3FileStorage_20230907171653.cs | 98 ----- .../Services/S3FileStorage_20230907171816.cs | 107 ------ .../Services/S3FileStorage_20230907171821.cs | 107 ------ .../Services/S3FileStorage_20230907171829.cs | 108 ------ .../Services/S3FileStorage_20230907171837.cs | 107 ------ .../Services/S3FileStorage_20230907171838.cs | 107 ------ .../Services/S3FileStorage_20230908125336.cs | 107 ------ .../Services/S3WriteStream_20230906141540.cs | 76 ---- .../Services/S3WriteStream_20230907104130.cs | 76 ---- .../Services/S3WriteStream_20230907104133.cs | 76 ---- .../Services/S3WriteStream_20230907130227.cs | 77 ---- .../Services/S3WriteStream_20230907130326.cs | 79 ---- .../Services/S3WriteStream_20230907130328.cs | 78 ---- .../Services/S3WriteStream_20230907130334.cs | 78 ---- .../Services/S3WriteStream_20230907130427.cs | 87 ----- .../Services/S3WriteStream_20230907130442.cs | 75 ---- .../Services/S3WriteStream_20230907130507.cs | 108 ------ .../Services/S3WriteStream_20230907130523.cs | 108 ------ .../Services/S3WriteStream_20230907130903.cs | 106 ------ .../Services/S3WriteStream_20230907130942.cs | 107 ------ .../Services/S3WriteStream_20230907130949.cs | 108 ------ .../Services/S3WriteStream_20230907131020.cs | 108 ------ .../Services/S3WriteStream_20230907131032.cs | 108 ------ .../Services/S3WriteStream_20230907131048.cs | 109 ------ .../Services/S3WriteStream_20230907131526.cs | 110 ------ .../Services/S3WriteStream_20230907131529.cs | 113 ------ .../Services/S3WriteStream_20230907131704.cs | 116 ------ .../Services/S3WriteStream_20230907131835.cs | 114 ------ .../Services/S3WriteStream_20230907131840.cs | 113 ------ .../Services/S3WriteStream_20230907131855.cs | 114 ------ .../Services/S3WriteStream_20230907131914.cs | 115 ------ .../Services/S3WriteStream_20230907131918.cs | 115 ------ .../Services/S3WriteStream_20230907132115.cs | 120 ------ .../Services/S3WriteStream_20230907132117.cs | 120 ------ .../Services/S3WriteStream_20230907132230.cs | 120 ------ .../Services/S3WriteStream_20230907132231.cs | 120 ------ .../Services/S3WriteStream_20230907132240.cs | 120 ------ .../Services/S3WriteStream_20230907132242.cs | 120 ------ .../Services/S3WriteStream_20230907132338.cs | 120 ------ .../Services/S3WriteStream_20230907132350.cs | 125 ------- .../Services/S3WriteStream_20230907132400.cs | 125 ------- .../Services/S3WriteStream_20230907132417.cs | 125 ------- .../Services/S3WriteStream_20230907132419.cs | 124 ------- .../Services/S3WriteStream_20230907132431.cs | 125 ------- .../Services/S3WriteStream_20230907132432.cs | 124 ------- .../Services/S3WriteStream_20230907132433.cs | 124 ------- .../Services/S3WriteStream_20230907132616.cs | 125 ------- .../Services/S3WriteStream_20230907132630.cs | 125 ------- .../Services/S3WriteStream_20230907132632.cs | 125 ------- .../Services/S3WriteStream_20230907132832.cs | 126 ------- .../Services/S3WriteStream_20230907132840.cs | 126 ------- .../Services/S3WriteStream_20230907133744.cs | 126 ------- .../Services/S3WriteStream_20230907140811.cs | 127 ------- .../Services/S3WriteStream_20230907140817.cs | 151 -------- .../Services/S3WriteStream_20230907140826.cs | 151 -------- .../Services/S3WriteStream_20230907141040.cs | 155 -------- .../Services/S3WriteStream_20230907141052.cs | 155 -------- .../Services/S3WriteStream_20230907141053.cs | 155 -------- .../Services/S3WriteStream_20230907141054.cs | 155 -------- .../Services/S3WriteStream_20230907141056.cs | 155 -------- .../Services/S3WriteStream_20230907141058.cs | 155 -------- .../Services/S3WriteStream_20230907141059.cs | 155 -------- .../Services/S3WriteStream_20230907141100.cs | 155 -------- .../Services/S3WriteStream_20230907141102.cs | 155 -------- .../Services/S3WriteStream_20230907142255.cs | 153 -------- .../Services/S3WriteStream_20230907171436.cs | 155 -------- .../Services/S3WriteStream_20230907171441.cs | 155 -------- .../Services/S3WriteStream_20230907171450.cs | 155 -------- .../Services/S3WriteStream_20230907171526.cs | 155 -------- .../Services/S3WriteStream_20230907171534.cs | 155 -------- .../Services/S3WriteStream_20230907171604.cs | 155 -------- .../Services/S3WriteStream_20230907171626.cs | 155 -------- .../Services/S3WriteStream_20230907171627.cs | 155 -------- .../Services/S3WriteStream_20230907171628.cs | 155 -------- .../Services/S3WriteStream_20230907171630.cs | 155 -------- .../Services/S3WriteStream_20230907171636.cs | 155 -------- .../Services/S3WriteStream_20230907171638.cs | 155 -------- .../Services/S3WriteStream_20230907171720.cs | 161 -------- .../Services/S3WriteStream_20230907171734.cs | 161 -------- .../Services/S3WriteStream_20230907171736.cs | 161 -------- .../Services/S3WriteStream_20230907172014.cs | 162 --------- .../Services/S3WriteStream_20230907172018.cs | 163 --------- .../Services/S3WriteStream_20230907172019.cs | 163 --------- .../Services/S3WriteStream_20230907172038.cs | 163 --------- .../Services/S3WriteStream_20230907172039.cs | 163 --------- .../Services/S3WriteStream_20230908125336.cs | 163 --------- .../Services/S3WriteStream_20230908125337.cs | 163 --------- .../Services/S3WriteStream_20230908125344.cs | 163 --------- .../SharedFileService_20230906141540.cs | 92 ----- .../SharedFileService_20230907104153.cs | 92 ----- .../SharedFileService_20230907134349.cs | 92 ----- .../SharedFileService_20230907134355.cs | 92 ----- .../SharedFileService_20230907171909.cs | 95 ----- .../SharedFileService_20230907171920.cs | 97 ----- .../SharedFileService_20230907171930.cs | 98 ----- .../SharedFileService_20230907171931.cs | 98 ----- .../SharedFileService_20230907171955.cs | 98 ----- .../SharedFileService_20230907171956.cs | 98 ----- .../SharedFileService_20230908125406.cs | 96 ----- .../SharedFileService_20230908125407.cs | 96 ----- ...SmtTransferEngineService_20230907084301.cs | 181 --------- ...SmtTransferEngineService_20230908161128.cs | 183 ---------- ...SmtTransferEngineService_20230908161129.cs | 183 ---------- ...SmtTransferEngineService_20230908161150.cs | 183 ---------- ...SmtTransferEngineService_20230908161950.cs | 183 ---------- ...SmtTransferEngineService_20230908162047.cs | 184 ---------- ...SmtTransferEngineService_20230908162317.cs | 188 ---------- ...SmtTransferEngineService_20230908162334.cs | 188 ---------- ...SmtTransferEngineService_20230908162336.cs | 188 ---------- ...SmtTransferEngineService_20230908162356.cs | 188 ---------- ...SmtTransferEngineService_20230908162358.cs | 188 ---------- ...SmtTransferEngineService_20230911083245.cs | 190 ---------- ...SmtTransferEngineService_20230911083247.cs | 190 ---------- ...SmtTransferEngineService_20230911083620.cs | 189 ---------- ...SmtTransferEngineService_20230911083625.cs | 189 ---------- ...SmtTransferEngineService_20230911083627.cs | 189 ---------- ...SmtTransferEngineService_20230911083628.cs | 189 ---------- ...SmtTransferEngineService_20230911083939.cs | 190 ---------- ...SmtTransferEngineService_20230911084126.cs | 192 ---------- ...SmtTransferEngineService_20230911084129.cs | 188 ---------- ...SmtTransferEngineService_20230911084135.cs | 188 ---------- ...SmtTransferEngineService_20230911084136.cs | 188 ---------- ...SmtTransferEngineService_20230911084149.cs | 188 ---------- ...SmtTransferEngineService_20230911084150.cs | 188 ---------- ...SmtTransferEngineService_20230911084151.cs | 188 ---------- ...SmtTransferEngineService_20230911084152.cs | 188 ---------- ...SmtTransferEngineService_20230911084153.cs | 188 ---------- ...SmtTransferEngineService_20230911084154.cs | 188 ---------- ...SmtTransferEngineService_20230911084155.cs | 188 ---------- ...SmtTransferEngineService_20230911084156.cs | 188 ---------- ...SmtTransferEngineService_20230911084157.cs | 188 ---------- ...SmtTransferEngineService_20230911084158.cs | 188 ---------- ...SmtTransferEngineService_20230911084159.cs | 188 ---------- ...SmtTransferEngineService_20230911084200.cs | 188 ---------- ...SmtTransferEngineService_20230911084201.cs | 188 ---------- ...SmtTransferEngineService_20230911084202.cs | 188 ---------- ...SmtTransferEngineService_20230911084203.cs | 188 ---------- ...SmtTransferEngineService_20230911084204.cs | 188 ---------- ...SmtTransferEngineService_20230911084205.cs | 188 ---------- ...SmtTransferEngineService_20230911084231.cs | 189 ---------- ...SmtTransferEngineService_20230911084236.cs | 191 ---------- ...SmtTransferEngineService_20230911084237.cs | 190 ---------- ...SmtTransferEngineService_20230911084410.cs | 190 ---------- ...SmtTransferEngineService_20230911084539.cs | 188 ---------- ...SmtTransferEngineService_20230911084556.cs | 185 ---------- ...SmtTransferEngineService_20230911084557.cs | 184 ---------- ...SmtTransferEngineService_20230911084610.cs | 185 ---------- ...SmtTransferEngineService_20230911084658.cs | 184 ---------- ...SmtTransferEngineService_20230911084700.cs | 184 ---------- ...SmtTransferEngineService_20230911084701.cs | 184 ---------- ...SmtTransferEngineService_20230911084711.cs | 184 ---------- ...SmtTransferEngineService_20230911084713.cs | 184 ---------- ...SmtTransferEngineService_20230911084838.cs | 184 ---------- ...SmtTransferEngineService_20230911084843.cs | 184 ---------- ...SmtTransferEngineService_20230911084844.cs | 184 ---------- ...SmtTransferEngineService_20230911084845.cs | 184 ---------- ...SmtTransferEngineService_20230911084846.cs | 184 ---------- ...SmtTransferEngineService_20230911085524.cs | 173 --------- ...SmtTransferEngineService_20230911085526.cs | 173 --------- ...SmtTransferEngineService_20230911085550.cs | 175 --------- ...SmtTransferEngineService_20230911085551.cs | 174 --------- ...SmtTransferEngineService_20230911085557.cs | 175 --------- ...SmtTransferEngineService_20230911085558.cs | 174 --------- ...SmtTransferEngineService_20230911085704.cs | 173 --------- ...SmtTransferEngineService_20230911085706.cs | 173 --------- ...SmtTransferEngineService_20230911085712.cs | 173 --------- ...SmtTransferEngineService_20230911085757.cs | 173 --------- ...SmtTransferEngineService_20230911085803.cs | 173 --------- ...SmtTransferEngineService_20230911085805.cs | 173 --------- ...SmtTransferEngineService_20230911085809.cs | 173 --------- ...slationEngineServiceBase_20230907084301.cs | 212 ----------- ...slationEngineServiceBase_20230911085214.cs | 219 ----------- ...slationEngineServiceBase_20230911085221.cs | 219 ----------- ...slationEngineServiceBase_20230911085339.cs | 227 ------------ ...slationEngineServiceBase_20230911085341.cs | 227 ------------ ...slationEngineServiceBase_20230911085421.cs | 227 ------------ ...slationEngineServiceBase_20230911085505.cs | 227 ------------ ...slationEngineServiceBase_20230911085623.cs | 230 ------------ ...slationEngineServiceBase_20230911085631.cs | 227 ------------ ...slationEngineServiceBase_20230911085641.cs | 228 ------------ ...slationEngineServiceBase_20230911085646.cs | 228 ------------ ...slationEngineServiceBase_20230911085651.cs | 228 ------------ ...slationEngineServiceBase_20230911085722.cs | 228 ------------ ...slationEngineServiceBase_20230911085823.cs | 228 ------------ .../Usings_20230907084316.cs | 47 --- .../Usings_20230907131743.cs | 47 --- .../Usings_20230907141855.cs | 49 --- .../Usings_20230907141856.cs | 48 --- .../Usings_20230907141900.cs | 48 --- .../Usings_20230907141901.cs | 48 --- ...rMLNmtEngineServiceTests_20230906141540.cs | 192 ---------- ...rMLNmtEngineServiceTests_20230907185104.cs | 192 ---------- 220 files changed, 35526 deletions(-) delete mode 100644 .history/.gitignore_20230911115750 delete mode 100644 .history/.gitignore_20230911115828 delete mode 100644 .history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs delete mode 100644 .history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs delete mode 100644 .history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs delete mode 100644 .history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs diff --git a/.history/.gitignore_20230911115750 b/.history/.gitignore_20230911115750 deleted file mode 100644 index e1f430ccb..000000000 --- a/.history/.gitignore_20230911115750 +++ /dev/null @@ -1,51 +0,0 @@ - -#ignore thumbnails created by windows -Thumbs.db -#Ignore files build by Visual Studio -*.obj -*.exe -*.pdb -*.user -*.aps -*.pch -*.vspscc -*_i.c -*_p.c -*.ncb -*.suo -*.tlb -*.tlh -*.bak -*.cache -*.ilk -*.log -[Bb]in -[Dd]ebug*/ -*.lib -*.sbr -obj/ -[Rr]elease*/ -_ReSharper*/ -[Tt]est[Rr]esult* - -*.Resharper -*.DotSettings -*- Copy* -packages/* -!packages/repositories.config -lib/ -*.files -*.VC.db -*.VC.opendb -*.lock.json -.vs -appsettings.user.json -artifacts -!**/Tes/release/ -thot-new-model.zip -*.tgz -out/ -src/sentencepiece4c/build/ -src/sentencepiece4cbuild/ -!samples/**/release -**/*.feature.cs diff --git a/.history/.gitignore_20230911115828 b/.history/.gitignore_20230911115828 deleted file mode 100644 index fd44de4b5..000000000 --- a/.history/.gitignore_20230911115828 +++ /dev/null @@ -1,52 +0,0 @@ - -#ignore thumbnails created by windows -Thumbs.db -#Ignore files build by Visual Studio -*.obj -*.exe -*.pdb -*.user -*.aps -*.pch -*.vspscc -*_i.c -*_p.c -*.ncb -*.suo -*.tlb -*.tlh -*.bak -*.cache -*.ilk -*.log -[Bb]in -[Dd]ebug*/ -*.lib -*.sbr -obj/ -[Rr]elease*/ -_ReSharper*/ -[Tt]est[Rr]esult* -.history* - -*.Resharper -*.DotSettings -*- Copy* -packages/* -!packages/repositories.config -lib/ -*.files -*.VC.db -*.VC.opendb -*.lock.json -.vs -appsettings.user.json -artifacts -!**/Tes/release/ -thot-new-model.zip -*.tgz -out/ -src/sentencepiece4c/build/ -src/sentencepiece4cbuild/ -!samples/**/release -**/*.feature.cs diff --git a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs b/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs deleted file mode 100644 index a8c7f1fd4..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907084316.cs +++ /dev/null @@ -1,317 +0,0 @@ -using Microsoft.AspNetCore.Http; -using Serval.Translation.V1; - -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMachineBuilderExtensions -{ - public static IMachineBuilder AddServiceOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddServiceOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddClearMLNmtEngineOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddClearMLNmtEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddThotSmtModel( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder.AddThotSmtModel(); - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder.AddThotSmtModel(); - } - - public static IMachineBuilder AddTransferEngine(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddUnigramTruecaser(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddClearMLService(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); - - // workaround register satisfying the interface and as a hosted service. - builder.Services.AddSingleton(); - builder.Services.AddHostedService(p => p.GetRequiredService()); - - builder.Services - .AddHttpClient() - .AddTransientHttpErrorPolicy( - b => b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) - ); - - return builder; - } - - public static IMachineBuilder AddMongoBackgroundJobClient( - this IMachineBuilder builder, - string? connectionString = null - ) - { - builder.Services.AddHangfire( - c => - c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) - .UseSimpleAssemblyNameTypeSerializer() - .UseRecommendedSerializerSettings() - .UseMongoStorage( - connectionString ?? builder.Configuration.GetConnectionString("Hangfire"), - new MongoStorageOptions - { - MigrationOptions = new MongoMigrationOptions - { - MigrationStrategy = new MigrateMongoMigrationStrategy(), - BackupStrategy = new CollectionMongoBackupStrategy() - }, - CheckConnection = true, - CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, - } - ) - ); - builder.Services.AddHealthChecks().AddCheck(name: "Hangfire"); - return builder; - } - - public static IMachineBuilder AddBackgroundJobServer( - this IMachineBuilder builder, - IEnumerable? engineTypes = null - ) - { - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; - var queues = new List(); - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - queues.Add("smt_transfer"); - break; - case TranslationEngineType.Nmt: - builder.AddClearMLService(); - queues.Add("nmt"); - break; - } - } - - builder.Services.AddHangfireServer(o => - { - o.Queues = queues.ToArray(); - }); - return builder; - } - - public static IMachineBuilder AddMemoryDataAccess(this IMachineBuilder builder) - { - builder.Services.AddMemoryDataAccess(o => - { - o.AddRepository(); - o.AddRepository(); - o.AddRepository(); - }); - - return builder; - } - - public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, string? connectionString = null) - { - connectionString ??= builder.Configuration.GetConnectionString("Mongo"); - builder.Services.AddMongoDataAccess( - connectionString, - "SIL.Machine.AspNetCore.Models", - o => - { - o.AddRepository( - "translation_engines", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.EngineId) - ) - ) - ); - o.AddRepository( - "locks", - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("writerLock._id")) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("readerLocks._id")) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("writerQueue._id")) - ); - } - ); - o.AddRepository( - "train_segment_pairs", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) - ) - ) - ); - } - ); - builder.Services.AddHealthChecks().AddMongoDb(connectionString, name: "Mongo"); - - return builder; - } - - public static IMachineBuilder AddServalPlatformService( - this IMachineBuilder builder, - string? connectionString = null - ) - { - builder.Services.AddScoped(); - builder.Services - .AddGrpcClient(o => - { - o.Address = new Uri(connectionString ?? builder.Configuration.GetConnectionString("Serval")); - }) - .ConfigureChannel(o => - { - o.MaxRetryAttempts = null; - o.ServiceConfig = new ServiceConfig - { - MethodConfigs = - { - new MethodConfig - { - Names = { MethodName.Default }, - RetryPolicy = new RetryPolicy - { - MaxAttempts = 10, - InitialBackoff = TimeSpan.FromSeconds(1), - MaxBackoff = TimeSpan.FromSeconds(5), - BackoffMultiplier = 1.5, - RetryableStatusCodes = { StatusCode.Unavailable } - } - }, - new MethodConfig - { - Names = - { - new MethodName - { - Service = "serval.translation.v1.TranslationPlatformApi", - Method = "UpdateBuildStatus" - } - } - }, - } - }; - }); - - return builder; - } - - public static IMachineBuilder AddServalTranslationEngineService( - this IMachineBuilder builder, - string? connectionString = null, - IEnumerable? engineTypes = null - ) - { - builder.Services.AddGrpc(options => options.Interceptors.Add()); - builder.AddServalPlatformService(connectionString ?? builder.Configuration.GetConnectionString("Serval")); - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.Services.AddSingleton(); - builder.Services.AddHostedService(); - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - builder.Services.AddScoped(); - break; - case TranslationEngineType.Nmt: - builder.AddClearMLService(); - builder.Services.AddScoped(); - break; - } - } - builder.Services.AddGrpcHealthChecks(); - - return builder; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs b/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs deleted file mode 100644 index 739a72924..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions_20230907133310.cs +++ /dev/null @@ -1,317 +0,0 @@ -using Microsoft.AspNetCore.Http; -using Serval.Translation.V1; - -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMachineBuilderExtensions -{ - public static IMachineBuilder AddServiceOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddServiceOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddClearMLNmtEngineOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddClearMLNmtEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddThotSmtModel( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder.AddThotSmtModel(); - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder.AddThotSmtModel(); - } - - public static IMachineBuilder AddTransferEngine(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddUnigramTruecaser(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddClearMLService(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); - - // workaround register satisfying the interface and as a hosted service. - builder.Services.AddSingleton(); - builder.Services.AddHostedService(p => p.GetRequiredService()); - - builder.Services - .AddHttpClient() - .AddTransientHttpErrorPolicy( - b => b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) - ); - - return builder; - } - - public static IMachineBuilder AddMongoBackgroundJobClient( - this IMachineBuilder builder, - string? connectionString = null - ) - { - builder.Services.AddHangfire( - c => - c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) - .UseSimpleAssemblyNameTypeSerializer() - .UseRecommendedSerializerSettings() - .UseMongoStorage( - connectionString ?? builder.Configuration.GetConnectionString("Hangfire"), - new MongoStorageOptions - { - MigrationOptions = new MongoMigrationOptions - { - MigrationStrategy = new MigrateMongoMigrationStrategy(), - BackupStrategy = new CollectionMongoBackupStrategy() - }, - CheckConnection = true, - CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, - } - ) - ); - builder.Services.AddHealthChecks().AddCheck(name: "Hangfire"); - return builder; - } - - public static IMachineBuilder AddBackgroundJobServer( - this IMachineBuilder builder, - IEnumerable? engineTypes = null - ) - { - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; - var queues = new List(); - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - queues.Add("smt_transfer"); - break; - case TranslationEngineType.Nmt: - builder.AddClearMLService(); - queues.Add("nmt"); - break; - } - } - - builder.Services.AddHangfireServer(o => - { - o.Queues = queues.ToArray(); - }); - return builder; - } - - public static IMachineBuilder AddMemoryDataAccess(this IMachineBuilder builder) - { - builder.Services.AddMemoryDataAccess(o => - { - o.AddRepository(); - o.AddRepository(); - o.AddRepository(); - }); - - return builder; - } - - public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, string? connectionString = null) - { - connectionString ??= builder.Configuration.GetConnectionString("Mongo"); - builder.Services.AddMongoDataAccess( - connectionString, - "SIL.Machine.AspNetCore.Models", - o => - { - o.AddRepository( - "translation_engines", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.EngineId) - ) - ) - ); - o.AddRepository( - "locks", - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("writerLock._id")) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("readerLocks._id")) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel(Builders.IndexKeys.Ascending("writerQueue._id")) - ); - } - ); - o.AddRepository( - "train_segment_pairs", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) - ) - ) - ); - } - ); - builder.Services.AddHealthChecks().AddMongoDb(connectionString, name: "Mongo"); - - return builder; - } - - public static IMachineBuilder AddServalPlatformService( - this IMachineBuilder builder, - string? connectionString = null - ) - { - builder.Services.AddScoped(); - builder.Services - .AddGrpcClient(o => - { - o.Address = new Uri(connectionString ?? builder.Configuration.GetConnectionString("Serval")); - }) - .ConfigureChannel(o => - { - o.MaxRetryAttempts = null; - o.ServiceConfig = new ServiceConfig - { - MethodConfigs = - { - new MethodConfig - { - Names = { MethodName.Default }, - RetryPolicy = new Grpc.Net.Client.Configuration.RetryPolicy - { - MaxAttempts = 10, - InitialBackoff = TimeSpan.FromSeconds(1), - MaxBackoff = TimeSpan.FromSeconds(5), - BackoffMultiplier = 1.5, - RetryableStatusCodes = { StatusCode.Unavailable } - } - }, - new MethodConfig - { - Names = - { - new MethodName - { - Service = "serval.translation.v1.TranslationPlatformApi", - Method = "UpdateBuildStatus" - } - } - }, - } - }; - }); - - return builder; - } - - public static IMachineBuilder AddServalTranslationEngineService( - this IMachineBuilder builder, - string? connectionString = null, - IEnumerable? engineTypes = null - ) - { - builder.Services.AddGrpc(options => options.Interceptors.Add()); - builder.AddServalPlatformService(connectionString ?? builder.Configuration.GetConnectionString("Serval")); - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? new[] { TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt }; - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.Services.AddSingleton(); - builder.Services.AddHostedService(); - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - builder.Services.AddScoped(); - break; - case TranslationEngineType.Nmt: - builder.AddClearMLService(); - builder.Services.AddScoped(); - break; - } - } - builder.Services.AddGrpcHealthChecks(); - - return builder; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907084316.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907095831.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs deleted file mode 100644 index 9e7eed761..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135725.cs +++ /dev/null @@ -1,343 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await sourceTrainWriter.BaseStream.DisposeAsync(); - - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs deleted file mode 100644 index e41df89d3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135737.cs +++ /dev/null @@ -1,342 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await targetTrainWriter.BaseStream.DisposeAsync(); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs deleted file mode 100644 index e41df89d3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135738.cs +++ /dev/null @@ -1,342 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await targetTrainWriter.BaseStream.DisposeAsync(); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs deleted file mode 100644 index e41df89d3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907135739.cs +++ /dev/null @@ -1,342 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await targetTrainWriter.BaseStream.DisposeAsync(); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs deleted file mode 100644 index e41df89d3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140302.cs +++ /dev/null @@ -1,342 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await targetTrainWriter.BaseStream.DisposeAsync(); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs deleted file mode 100644 index bf3981d14..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140611.cs +++ /dev/null @@ -1,342 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - await sourceTrainWriter.BaseStream.DisposeAsync(); - await targetTrainWriter.BaseStream.DisposeAsync(); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs deleted file mode 100644 index a26cda901..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140709.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140719.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907140720.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141110.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141112.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs deleted file mode 100644 index 918f40b90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineBuildJob_20230907141905.cs +++ /dev/null @@ -1,340 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineBuildJob -{ - private readonly IPlatformService _platformService; - private readonly IRepository _engines; - private readonly ILogger _logger; - private readonly IClearMLService _clearMLService; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - private readonly ICorpusService _corpusService; - - public ClearMLNmtEngineBuildJob( - IPlatformService platformService, - IRepository engines, - ILogger logger, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor options, - ICorpusService corpusService - ) - { - _platformService = platformService; - _engines = engines; - _logger = logger; - _clearMLService = clearMLService; - _sharedFileService = sharedFileService; - _options = options; - _corpusService = corpusService; - } - - [Queue("nmt")] - [AutomaticRetry(Attempts = 0)] - public async Task RunAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - string? clearMLProjectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (clearMLProjectId is null) - return; - - try - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken: cancellationToken - ); - if (engine is null || engine.IsCanceled) - throw new OperationCanceledException(); - - int corpusSize; - if (engine.BuildState is BuildState.Pending) - corpusSize = await WriteDataFilesAsync(buildId, corpora, cancellationToken); - else - corpusSize = GetCorpusSize(corpora); - - string clearMLTaskId; - ClearMLTask? clearMLTask = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (clearMLTask is null) - { - clearMLTaskId = await _clearMLService.CreateTaskAsync( - buildId, - clearMLProjectId, - engineId, - engine.SourceLanguage, - engine.TargetLanguage, - _sharedFileService.GetBaseUri().ToString(), - cancellationToken - ); - await _clearMLService.EnqueueTaskAsync(clearMLTaskId, CancellationToken.None); - } - else - { - clearMLTaskId = clearMLTask.Id; - } - - int lastIteration = 0; - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - - clearMLTask = await _clearMLService.GetTaskByIdAsync(clearMLTaskId, cancellationToken); - if (clearMLTask is null) - throw new InvalidOperationException("The ClearML task does not exist."); - - if ( - engine.BuildState == BuildState.Pending - && clearMLTask.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId && !e.IsCanceled, - u => u.Set(e => e.BuildState, BuildState.Active), - cancellationToken: cancellationToken - ); - if (engine is null) - throw new OperationCanceledException(); - await _platformService.BuildStartedAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build started ({0})", buildId); - } - - switch (clearMLTask.Status) - { - case ClearMLTaskStatus.InProgress: - case ClearMLTaskStatus.Completed: - if (lastIteration != clearMLTask.LastIteration) - { - await _platformService.UpdateBuildStatusAsync(buildId, clearMLTask.LastIteration); - lastIteration = clearMLTask.LastIteration; - } - break; - case ClearMLTaskStatus.Stopped: - // This could have been triggered from the ClearML UI, so set IsCanceled to true. - await _engines.UpdateAsync( - e => e.EngineId == engineId && !e.IsCanceled, - u => u.Set(e => e.IsCanceled, true), - cancellationToken: CancellationToken.None - ); - throw new OperationCanceledException(); - case ClearMLTaskStatus.Failed: - throw new InvalidOperationException(clearMLTask.StatusReason); - } - if (clearMLTask.Status is ClearMLTaskStatus.Completed) - break; - await Task.Delay(_options.CurrentValue.BuildPollingTimeout, cancellationToken); - } - - // The ClearML task has successfully completed, so insert the generated pretranslations into the database. - await InsertPretranslationsAsync(engineId, buildId, cancellationToken); - - IReadOnlyDictionary metrics = await _clearMLService.GetTaskMetricsAsync( - clearMLTaskId, - CancellationToken.None - ); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Inc(e => e.BuildRevision) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (!metrics.TryGetValue("bleu", out double confidence)) - confidence = 0; - - await _platformService.BuildCompletedAsync( - buildId, - corpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - _logger.LogInformation("Build completed in {0}s ({1})", clearMLTask.ActiveDuration, buildId); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - CancellationToken.None - ); - if (engine is null || engine.IsCanceled) - { - // This is an actual cancellation triggered by an API call. - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, CancellationToken.None); - if (task is not null) - await _clearMLService.StopTaskAsync(task.Id, CancellationToken.None); - - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - - bool buildStarted = await _engines.ExistsAsync( - e => e.EngineId == engineId && e.BuildId == buildId && e.BuildState == BuildState.Active, - CancellationToken.None - ); - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - if (buildStarted) - { - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - _logger.LogInformation("Build canceled ({0})", buildId); - } - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - await _platformService.BuildRestartingAsync(buildId, CancellationToken.None); - } - - throw; - } - catch (Exception e) - { - _logger.LogError(0, e, $"Build faulted ({buildId}) because of exception {e.GetType().Name}:{e.Message}."); - - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e2) - { - _logger.LogError( - $"Unable to access S3 bucket to delete clearml job {buildId} because it threw the exception {e2.GetType().Name}:{e2.Message}." - ); - } - - await _engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - u => - u.Set(e => e.BuildState, BuildState.None) - .Set(e => e.IsCanceled, false) - .Unset(e => e.JobId) - .Unset(e => e.BuildId), - cancellationToken: CancellationToken.None - ); - - await _platformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - throw; - } - } - - private async Task WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - await using var sourceTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken) - ); - await using var targetTrainWriter = new StreamWriter( - await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken) - ); - - int corpusSize = 0; - async IAsyncEnumerable ProcessRowsAsync() - { - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows( - targetCorpus, - allSourceRows: true, - allTargetRows: true - ); - - foreach (ParallelTextRow row in parallelCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - if ( - (corpus.PretranslateAll || corpus.PretranslateTextIds.Contains(row.TextId)) - && row.SourceSegment.Count > 0 - && row.TargetSegment.Count == 0 - ) - { - yield return new Pretranslation - { - CorpusId = corpus.Id, - TextId = row.TextId, - Refs = row.TargetRefs.Select(r => r.ToString()!).ToList(), - Translation = row.SourceText - }; - } - if (!row.IsEmpty) - corpusSize++; - } - } - } - - await using var sourcePretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - await JsonSerializer.SerializeAsync( - sourcePretranslateStream, - ProcessRowsAsync(), - new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken: cancellationToken - ); - return corpusSize; - } - - private int GetCorpusSize(IReadOnlyList corpora) - { - int corpusSize = 0; - foreach (Corpus corpus in corpora) - { - ITextCorpus sourceCorpus = _corpusService.CreateTextCorpus(corpus.SourceFiles); - ITextCorpus targetCorpus = _corpusService.CreateTextCorpus(corpus.TargetFiles); - - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - corpusSize += parallelCorpus.Count(includeEmpty: false); - } - return corpusSize; - } - - private async Task InsertPretranslationsAsync(string engineId, string buildId, CancellationToken cancellationToken) - { - await using var targetPretranslateStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - targetPretranslateStream, - new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }, - cancellationToken - ) - .OfType(); - - await _platformService.InsertPretranslationsAsync(engineId, pretranslations, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs deleted file mode 100644 index 42eb064e1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230906141540.cs +++ /dev/null @@ -1,51 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineService : TranslationEngineServiceBase -{ - private readonly IClearMLService _clearMLService; - - public ClearMLNmtEngineService( - IBackgroundJobClient jobClient, - IPlatformService platformService, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IRepository engines, - IClearMLService clearMLService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _clearMLService = clearMLService; - } - - public override TranslationEngineType Type => TranslationEngineType.Nmt; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - await _clearMLService.CreateProjectAsync(engineId, engineName, cancellationToken: CancellationToken.None); - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - string? projectId = await _clearMLService.GetProjectIdAsync(engineId, CancellationToken.None); - if (projectId is not null) - await _clearMLService.DeleteProjectAsync(projectId, CancellationToken.None); - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs deleted file mode 100644 index 42eb064e1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085826.cs +++ /dev/null @@ -1,51 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineService : TranslationEngineServiceBase -{ - private readonly IClearMLService _clearMLService; - - public ClearMLNmtEngineService( - IBackgroundJobClient jobClient, - IPlatformService platformService, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IRepository engines, - IClearMLService clearMLService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _clearMLService = clearMLService; - } - - public override TranslationEngineType Type => TranslationEngineType.Nmt; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - await _clearMLService.CreateProjectAsync(engineId, engineName, cancellationToken: CancellationToken.None); - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - string? projectId = await _clearMLService.GetProjectIdAsync(engineId, CancellationToken.None); - if (projectId is not null) - await _clearMLService.DeleteProjectAsync(projectId, CancellationToken.None); - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs b/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs deleted file mode 100644 index 42eb064e1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/ClearMLNmtEngineService_20230911085829.cs +++ /dev/null @@ -1,51 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLNmtEngineService : TranslationEngineServiceBase -{ - private readonly IClearMLService _clearMLService; - - public ClearMLNmtEngineService( - IBackgroundJobClient jobClient, - IPlatformService platformService, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IRepository engines, - IClearMLService clearMLService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _clearMLService = clearMLService; - } - - public override TranslationEngineType Type => TranslationEngineType.Nmt; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - await _clearMLService.CreateProjectAsync(engineId, engineName, cancellationToken: CancellationToken.None); - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - string? projectId = await _clearMLService.GetProjectIdAsync(engineId, CancellationToken.None); - if (projectId is not null) - await _clearMLService.DeleteProjectAsync(projectId, CancellationToken.None); - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs deleted file mode 100644 index 56bbc9829..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230906141540.cs +++ /dev/null @@ -1,95 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - return Task.FromResult( - new BufferedStream(new S3WriteStream(_client, objectId, _bucketName), 1024 * 1024 * 100) - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs deleted file mode 100644 index 56bbc9829..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907104119.cs +++ /dev/null @@ -1,95 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - return Task.FromResult( - new BufferedStream(new S3WriteStream(_client, objectId, _bucketName), 1024 * 1024 * 100) - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs deleted file mode 100644 index aaf4a3f52..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130153.cs +++ /dev/null @@ -1,97 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - 1024 * 1024 * 5 - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs deleted file mode 100644 index 66204bcaa..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907130157.cs +++ /dev/null @@ -1,97 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - 1024 * 1024 * 5)); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs deleted file mode 100644 index 413860de7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132651.cs +++ /dev/null @@ -1,98 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - 1024 * 1024 * 5 - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs deleted file mode 100644 index 413860de7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907132652.cs +++ /dev/null @@ -1,98 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - 1024 * 1024 * 5 - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs deleted file mode 100644 index 1eeacbc76..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171652.cs +++ /dev/null @@ -1,98 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs deleted file mode 100644 index 1eeacbc76..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171653.cs +++ /dev/null @@ -1,98 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - - public S3FileStorage(string bucketName, string basePath, string accessKeyId, string secretAccessKey, string region) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs deleted file mode 100644 index 2ae672c3c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171816.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = Amazon.Runtime.RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs deleted file mode 100644 index 0619ee785..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171821.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs deleted file mode 100644 index b5eb883cb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171829.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId), - S3WriteStream.FiveMB, - _loggerFactory - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs deleted file mode 100644 index 9058da211..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171837.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId, _loggerFactory), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs deleted file mode 100644 index 9058da211..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230907171838.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId, _loggerFactory), - S3WriteStream.FiveMB - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs deleted file mode 100644 index ff8563a42..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3FileStorage_20230908125336.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : FileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config - { - RetryMode = RequestRetryMode.Standard, - MaxErrorRetry = 3, - RegionEndpoint = RegionEndpoint.GetBySystemName(region) - } - ); - - _bucketName = bucketName; - //Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level implementation - _basePath = basePath.EndsWith("/") ? basePath.Remove(basePath.Length - 1, 1) : basePath; - _basePath = _basePath.StartsWith("/") ? _basePath.Remove(0, 1) : _basePath; - _loggerFactory = loggerFactory; - } - - public override void Dispose() { } - - public override async Task Exists(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: path.EndsWith("/")), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public override async Task> Ls( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path, includeTrailingSlash: true), - MaxKeys = 1, - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key).ToList(); - } - - public override async Task OpenRead(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - GetObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {objectId} does not exist"); - return response.ResponseStream; - } - - public override async Task OpenWrite(string path, CancellationToken cancellationToken = default) - { - string objectId = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = objectId }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync(request); - return new BufferedStream( - new S3WriteStream(_client, objectId, _bucketName, response.UploadId, _loggerFactory), - S3WriteStream.MaxPartSize - ); - } - - public override async Task Rm(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - if (path is null) - throw new ArgumentNullException(nameof(path)); - string objectId = _basePath + Normalize(path); - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = objectId }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.OK)) - new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs deleted file mode 100644 index b2d5808f0..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230906141540.cs +++ /dev/null @@ -1,76 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _bucketName; - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _length = 0; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs deleted file mode 100644 index e27f3416a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104130.cs +++ /dev/null @@ -1,76 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services;S3Wri - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _bucketName; - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _length = 0; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs deleted file mode 100644 index b2d5808f0..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907104133.cs +++ /dev/null @@ -1,76 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _bucketName; - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _length = 0; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs deleted file mode 100644 index a4284a6fe..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130227.cs +++ /dev/null @@ -1,77 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _bucketName; - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _length = 0; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs deleted file mode 100644 index effbfca90..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130326.cs +++ /dev/null @@ -1,79 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs deleted file mode 100644 index 1fcc576e6..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130328.cs +++ /dev/null @@ -1,78 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs deleted file mode 100644 index 213ddd80a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130334.cs +++ /dev/null @@ -1,78 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - await transferUtility.UploadAsync(uploadRequest); - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs deleted file mode 100644 index 24f757833..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130427.cs +++ /dev/null @@ -1,87 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) - { - using Stream inputStream = new MemoryStream(buffer, offset, count); - using var transferUtility = new TransferUtility(_client); - var uploadRequest = new TransferUtilityUploadRequest - { - BucketName = _bucketName, - InputStream = inputStream, - Key = _key, - PartSize = count - }; - transferUtility.Upload(uploadRequest); - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - FilePosition = _length - }; - _length += count; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs deleted file mode 100644 index 04454309b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130442.cs +++ /dev/null @@ -1,75 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - FilePosition = _length - }; - _length += count; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - public override ValueTask DisposeAsync() - { - Dispose(disposing: false); - GC.SuppressFinalize(this); - return ValueTask.CompletedTask; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs deleted file mode 100644 index 46a493f7f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130507.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - FilePosition = _length - }; - _length += count; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs deleted file mode 100644 index 46a493f7f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130523.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - FilePosition = _length - }; - _length += count; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs deleted file mode 100644 index a5cdde5c5..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130903.cs +++ /dev/null @@ -1,106 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber - }; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs deleted file mode 100644 index 16f137a5a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130942.cs +++ /dev/null @@ -1,107 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber - }; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs deleted file mode 100644 index 7d72d7d4d..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907130949.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - Stream = ms - }; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs deleted file mode 100644 index 21f06731c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131020.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs deleted file mode 100644 index 21f06731c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131032.cs +++ /dev/null @@ -1,108 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - uploadResponses.Add(await _client.UploadPartAsync(request)); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs deleted file mode 100644 index c551af600..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131048.cs +++ /dev/null @@ -1,109 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - UploadPartResponse response = await _client.UploadPartAsync(request); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs deleted file mode 100644 index 859618376..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131526.cs +++ /dev/null @@ -1,110 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - UploadPartResponse response = await _client.UploadPartAsync(request); - if(response.HttpStatusCode != HttpStatusCode.OK) throw new HttpRequestException($"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}") - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs deleted file mode 100644 index 38f8cf6d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131529.cs +++ /dev/null @@ -1,113 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs deleted file mode 100644 index 4e5370496..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131704.cs +++ /dev/null @@ -1,116 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => _length; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - uploadRequest.StreamTransferProgress += new EventHandler( - UploadPartProgressEventCallback - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs deleted file mode 100644 index e89e96826..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131835.cs +++ /dev/null @@ -1,114 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private long _length; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler((_, e) => { }); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs deleted file mode 100644 index 7be03bf75..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131840.cs +++ /dev/null @@ -1,113 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler((_, e) => { }); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs deleted file mode 100644 index 785cf5858..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131855.cs +++ /dev/null @@ -1,114 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler((_, e) => { }); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs deleted file mode 100644 index 64918822a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131914.cs +++ /dev/null @@ -1,115 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = (new LoggerFactory()).CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler((_, e) => { }); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs deleted file mode 100644 index 3a05d9ab0..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907131918.cs +++ /dev/null @@ -1,115 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler((_, e) => { }); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs deleted file mode 100644 index 2d1c7f47a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132115.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs deleted file mode 100644 index 2d1c7f47a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132117.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception) - { - await Abort(); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort() - { - // Logging? - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs deleted file mode 100644 index 7c5630f4b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132230.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs deleted file mode 100644 index 7c5630f4b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132231.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception) - { - await Abort(); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs deleted file mode 100644 index 52ecc2ef3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132240.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs deleted file mode 100644 index 52ecc2ef3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132242.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs deleted file mode 100644 index cb6f6edb5..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132338.cs +++ /dev/null @@ -1,120 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs deleted file mode 100644 index d6bc7c0aa..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132350.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs deleted file mode 100644 index 81005c673..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132400.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Uploading part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs deleted file mode 100644 index bff5aa1d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132417.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs deleted file mode 100644 index ac7aa225c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132419.cs +++ /dev/null @@ -1,124 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - Dispose(disposing: false); - GC.SuppressFinalize(this); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs deleted file mode 100644 index 5f5b59245..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132431.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs deleted file mode 100644 index 530e928ca..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132432.cs +++ /dev/null @@ -1,124 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs deleted file mode 100644 index 530e928ca..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132433.cs +++ /dev/null @@ -1,124 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List uploadResponses = new(); - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs deleted file mode 100644 index 7e13f6a62..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132616.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs deleted file mode 100644 index 104dc7116..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132630.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs deleted file mode 100644 index 104dc7116..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132632.cs +++ /dev/null @@ -1,125 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs deleted file mode 100644 index a97937131..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132832.cs +++ /dev/null @@ -1,126 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs deleted file mode 100644 index a97937131..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907132840.cs +++ /dev/null @@ -1,126 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs deleted file mode 100644 index a97937131..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907133744.cs +++ /dev/null @@ -1,126 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs deleted file mode 100644 index 1024e4aec..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140811.cs +++ /dev/null @@ -1,127 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) { } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs deleted file mode 100644 index 4ef04e880..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140817.cs +++ /dev/null @@ -1,151 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs deleted file mode 100644 index e4c5eb276..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907140826.cs +++ /dev/null @@ -1,151 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client.CompleteMultipartUpload(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs deleted file mode 100644 index 9a03bb2e4..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141040.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141052.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141053.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141054.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141056.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141058.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141059.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141100.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs deleted file mode 100644 index df9aba354..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907141102.cs +++ /dev/null @@ -1,155 +0,0 @@ -using Nito.AsyncEx.Synchronous; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs deleted file mode 100644 index 584a0e09c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907142255.cs +++ /dev/null @@ -1,153 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs deleted file mode 100644 index 9b4f9cf96..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171436.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public static final int fiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs deleted file mode 100644 index d7b0b6982..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171441.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public final static int fiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs deleted file mode 100644 index a47284fb5..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171450.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public static const int fiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs deleted file mode 100644 index 50a58fafe..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171526.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int fiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs deleted file mode 100644 index c13ee9bc2..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171534.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FIVE_MB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs deleted file mode 100644 index c13ee9bc2..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171604.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FIVE_MB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs deleted file mode 100644 index 20cee4645..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171626.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMb = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs deleted file mode 100644 index 262f9a6c3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171627.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMbB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs deleted file mode 100644 index 9ea77dfee..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171628.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs deleted file mode 100644 index 9ea77dfee..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171630.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = 5 * 1024 * 1024 - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs deleted file mode 100644 index 6d25881bb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171636.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs deleted file mode 100644 index 6d25881bb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171638.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream(AmazonS3Client client, string key, string bucketName, string uploadId) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs deleted file mode 100644 index 95dd7899d..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171720.cs +++ /dev/null @@ -1,161 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = new LoggerFactory().CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs deleted file mode 100644 index 001301acb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171734.cs +++ /dev/null @@ -1,161 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs deleted file mode 100644 index 001301acb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907171736.cs +++ /dev/null @@ -1,161 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs deleted file mode 100644 index 7fc4c6f7a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172014.cs +++ /dev/null @@ -1,162 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs deleted file mode 100644 index afd96e3f9..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172018.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs deleted file mode 100644 index afd96e3f9..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172019.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await Abort(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - Abort(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await Abort(e); - } - } - - private async Task Abort(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs deleted file mode 100644 index 339ca3a26..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172038.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs deleted file mode 100644 index 339ca3a26..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230907172039.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int FiveMB = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = FiveMB - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs deleted file mode 100644 index fc2173053..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125336.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int MaxPartSize = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = MaxPartSize - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs deleted file mode 100644 index fc2173053..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125337.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int MaxPartSize = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = MaxPartSize - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs b/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs deleted file mode 100644 index fc2173053..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/S3WriteStream_20230908125344.cs +++ /dev/null @@ -1,163 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream : Stream -{ - private readonly AmazonS3Client _client; - private readonly string _key; - private readonly string _uploadId; - private readonly string _bucketName; - private readonly List _uploadResponses; - private readonly ILogger _logger; - - public const int MaxPartSize = 5 * 1024 * 1024; - - public S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory - ) - { - _client = client; - _key = key; - _bucketName = bucketName; - _uploadId = uploadId; - _logger = loggerFactory.CreateLogger(); - _uploadResponses = new List(); - } - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - try - { - using MemoryStream ms = new(buffer, offset, count); - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = ms, - PartSize = MaxPartSize - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug($"Transferred {e.TransferredBytes}/{e.TotalBytes}"); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - _uploadResponses.Add(response); - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - base.Dispose(disposing); - } - - public async override ValueTask DisposeAsync() - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception e) - { - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs deleted file mode 100644 index 526cf95fd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230906141540.cs +++ /dev/null @@ -1,92 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - - public SharedFileService(IOptions? options = null) - { - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs deleted file mode 100644 index 526cf95fd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907104153.cs +++ /dev/null @@ -1,92 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - - public SharedFileService(IOptions? options = null) - { - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs deleted file mode 100644 index a7a778353..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134349.cs +++ /dev/null @@ -1,92 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - - public SharedFileService(IOptions? options = null) - { - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - }} - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs deleted file mode 100644 index 526cf95fd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907134355.cs +++ /dev/null @@ -1,92 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - - public SharedFileService(IOptions? options = null) - { - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs deleted file mode 100644 index c7eb4f847..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171909.cs +++ /dev/null @@ -1,95 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) - { - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs deleted file mode 100644 index 53777aeaa..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171920.cs +++ /dev/null @@ -1,97 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs deleted file mode 100644 index c9f4aa6b1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171930.cs +++ /dev/null @@ -1,98 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs deleted file mode 100644 index c9f4aa6b1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171931.cs +++ /dev/null @@ -1,98 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(IOptions? options = null, ILoggerFactory loggerFactory) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs deleted file mode 100644 index 5804584c1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171955.cs +++ /dev/null @@ -1,98 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs deleted file mode 100644 index 5804584c1..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230907171956.cs +++ /dev/null @@ -1,98 +0,0 @@ -using SIL.Reporting; - -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs deleted file mode 100644 index cef73bbeb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125406.cs +++ /dev/null @@ -1,96 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs b/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs deleted file mode 100644 index cef73bbeb..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SharedFileService_20230908125407.cs +++ /dev/null @@ -1,96 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly FileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - Directory.CreateDirectory(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenRead(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWrite(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.Ls(path, recurse: true, cancellationToken); - foreach (string file in files) - await _fileStorage.Rm(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.Rm(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.Exists(path, cancellationToken); - } - - public Task> Ls( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.Ls(path, recurse, cancellationToken); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs deleted file mode 100644 index e7619bc63..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230907084301.cs +++ /dev/null @@ -1,181 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs deleted file mode 100644 index 804a21c2e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161128.cs +++ /dev/null @@ -1,183 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - if (state.CurrentBuildRevision == -1) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs deleted file mode 100644 index 804a21c2e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161129.cs +++ /dev/null @@ -1,183 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - if (state.CurrentBuildRevision == -1) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs deleted file mode 100644 index 804a21c2e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161150.cs +++ /dev/null @@ -1,183 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - if (state.CurrentBuildRevision == -1) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs deleted file mode 100644 index 68d6dc96e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908161950.cs +++ /dev/null @@ -1,183 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - if (state.CurrentBuildRevision == -1) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs deleted file mode 100644 index 38b27e746..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162047.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if (await Engines.GetAsync(e => e.Id == engineId && e.BuildId)) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs deleted file mode 100644 index 4c6134aa9..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162317.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - ( - await Engines.GetAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs deleted file mode 100644 index cf92eef1a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162334.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - ( - await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs deleted file mode 100644 index cf92eef1a..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162336.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - ( - await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs deleted file mode 100644 index 4672cf290..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162356.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - !( - await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs deleted file mode 100644 index 4672cf290..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230908162358.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - !( - await Engines.GetAllAsync(e => e.Id == engineId && e.BuildId != null && e.BuildState == BuildState.None) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs deleted file mode 100644 index f75ec89f0..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083245.cs +++ /dev/null @@ -1,190 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs deleted file mode 100644 index f75ec89f0..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083247.cs +++ /dev/null @@ -1,190 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine e; - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs deleted file mode 100644 index 22727b2d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083620.cs +++ /dev/null @@ -1,189 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs deleted file mode 100644 index 22727b2d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083625.cs +++ /dev/null @@ -1,189 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs deleted file mode 100644 index 22727b2d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083627.cs +++ /dev/null @@ -1,189 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs deleted file mode 100644 index 22727b2d7..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083628.cs +++ /dev/null @@ -1,189 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - if ( - !( - await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ) - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs deleted file mode 100644 index f5d00996f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911083939.cs +++ /dev/null @@ -1,190 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - if ( - !( - - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs deleted file mode 100644 index dd3aa1037..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084126.cs +++ /dev/null @@ -1,192 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach(var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if ( - !( - - ).Any() - ) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs deleted file mode 100644 index b773a87c2..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084129.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!(engines).Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084135.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084136.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084149.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084150.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084151.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084152.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084153.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084154.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084155.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084156.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084157.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084158.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084159.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084200.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084201.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084202.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084203.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084204.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs deleted file mode 100644 index ec235055c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084205.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs deleted file mode 100644 index 46ad52609..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084231.cs +++ /dev/null @@ -1,189 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Console.WriteLine("|||||||||||||||||||"); - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs deleted file mode 100644 index 9991f0c87..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084236.cs +++ /dev/null @@ -1,191 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Console.WriteLine("|||||||||||||||||||"); - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - Console.WriteLine("|||||||||||||||||||"); - - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs deleted file mode 100644 index 3574fa79f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084237.cs +++ /dev/null @@ -1,190 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Console.WriteLine("|||||||||||||||||||"); - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildRevision > 0 && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - Console.WriteLine("|||||||||||||||||||"); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs deleted file mode 100644 index 47ce83b1f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084410.cs +++ /dev/null @@ -1,190 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Console.WriteLine("|||||||||||||||||||"); - IEnumerable engines = await Engines.GetAllAsync( - e => e.Id == engineId && e.BuildState == BuildState.None - ); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - Console.WriteLine("|||||||||||||||||||"); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs deleted file mode 100644 index 234a20433..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084539.cs +++ /dev/null @@ -1,188 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - Console.WriteLine("|||||||||||||||||||"); - IEnumerable engines = await Engines.GetAllAsync(e => e.Id == engineId); - foreach (var eng in engines) - Console.WriteLine(JsonSerializer.Serialize(eng)); - Console.WriteLine("|||||||||||||||||||"); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs deleted file mode 100644 index 92d3d3d1e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084556.cs +++ /dev/null @@ -1,185 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs deleted file mode 100644 index 26186680c..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084557.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs deleted file mode 100644 index eaa649a10..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084610.cs +++ /dev/null @@ -1,185 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (!engines.Any()) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs deleted file mode 100644 index 76069459f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084658.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildRevision == 0 || engine.BuildState != BuildState.None || engine.Revision == 0) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs deleted file mode 100644 index 76069459f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084700.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildRevision == 0 || engine.BuildState != BuildState.None || engine.Revision == 0) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs deleted file mode 100644 index 76069459f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084701.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildRevision == 0 || engine.BuildState != BuildState.None || engine.Revision == 0) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs deleted file mode 100644 index b73ba6ead..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084711.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.Revision == 0) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs deleted file mode 100644 index b73ba6ead..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084713.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.Revision == 0) - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs deleted file mode 100644 index ebb403ba9..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084838.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.Revision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs deleted file mode 100644 index 296b73779..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084843.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs deleted file mode 100644 index 296b73779..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084844.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs deleted file mode 100644 index 296b73779..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084845.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs deleted file mode 100644 index 296b73779..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911084846.cs +++ /dev/null @@ -1,184 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs deleted file mode 100644 index 3ab3f9d3f..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085524.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs deleted file mode 100644 index e4be91c25..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085526.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs deleted file mode 100644 index 91a738627..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085550.cs +++ /dev/null @@ -1,175 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs deleted file mode 100644 index e4eba1d38..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085551.cs +++ /dev/null @@ -1,174 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs deleted file mode 100644 index b4dde802b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085557.cs +++ /dev/null @@ -1,175 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs deleted file mode 100644 index 0eae7ff8b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085558.cs +++ /dev/null @@ -1,174 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - await ThrowExceptionIfEngineIsNotBuilt(engineId, cancellationToken); - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs deleted file mode 100644 index 6bc79f5bd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085704.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs deleted file mode 100644 index 6bc79f5bd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085706.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs deleted file mode 100644 index 8538f7d87..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085712.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await G(engineetBuiltEngineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs deleted file mode 100644 index 6bc79f5bd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085757.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs deleted file mode 100644 index 67a1023da..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085803.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngine(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs deleted file mode 100644 index 2cfccdc7e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085805.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs b/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs deleted file mode 100644 index 2cfccdc7e..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService_20230911085809.cs +++ /dev/null @@ -1,173 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService : TranslationEngineServiceBase -{ - private readonly IRepository _trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService; - - public SmtTransferEngineService( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService - ) - : base(jobClient, lockFactory, platformService, dataAccessContext, engines) - { - _trainSegmentPairs = trainSegmentPairs; - _stateService = stateService; - } - - public override TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public override async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await base.CreateAsync(engineId, engineName, sourceLanguage, targetLanguage, cancellationToken); - - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - state.InitNew(); - } - } - - public override async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await base.DeleteAsync(engineId, cancellationToken); - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - // ensure that there is no build running before unloading - string? buildId = await CancelBuildInternalAsync(engineId, CancellationToken.None); - if (buildId is not null) - await WaitForBuildToFinishAsync(engineId, buildId, CancellationToken.None); - - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - } - - public override async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public override async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public override async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState is BuildState.Active) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engine.Id, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - cancellationToken - ); - } - - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, cancellationToken); - await PlatformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - if (engine.BuildState is BuildState.Active) - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public override async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public override async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - SmtTransferEngineState state = _stateService.Get(engineId); - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - state.LastUsedTime = DateTime.UtcNow; - } - } - - protected override Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ) - { - // Token "None" is used here because hangfire injects the proper cancellation token - return r => r.RunAsync(engineId, buildId, corpora, CancellationToken.None); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs deleted file mode 100644 index 0f3d710af..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230907084301.cs +++ /dev/null @@ -1,212 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs deleted file mode 100644 index 41ff71d4b..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085214.cs +++ /dev/null @@ -1,219 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs deleted file mode 100644 index d7acd2bfd..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085221.cs +++ /dev/null @@ -1,219 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs deleted file mode 100644 index 2d7e5e928..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085339.cs +++ /dev/null @@ -1,227 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs deleted file mode 100644 index 2d7e5e928..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085341.cs +++ /dev/null @@ -1,227 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs deleted file mode 100644 index 2d7e5e928..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085421.cs +++ /dev/null @@ -1,227 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs deleted file mode 100644 index 2d7e5e928..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085505.cs +++ /dev/null @@ -1,227 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs deleted file mode 100644 index 8c2e0cbc3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085623.cs +++ /dev/null @@ -1,230 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task ThrowExceptionIfEngineIsNotBuilt( - string engineId, - CancellationToken cancellationToken - ) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs deleted file mode 100644 index f310e85f3..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085631.cs +++ /dev/null @@ -1,227 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs deleted file mode 100644 index dc702f8de..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085641.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs deleted file mode 100644 index dc702f8de..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085646.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs deleted file mode 100644 index d6511bf11..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085651.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs deleted file mode 100644 index dc702f8de..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085722.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs b/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs deleted file mode 100644 index dc702f8de..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Services/TranslationEngineServiceBase_20230911085823.cs +++ /dev/null @@ -1,228 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class TranslationEngineServiceBase : ITranslationEngineService -{ - private readonly IBackgroundJobClient _jobClient; - - protected TranslationEngineServiceBase( - IBackgroundJobClient jobClient, - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines - ) - { - _jobClient = jobClient; - LockFactory = lockFactory; - PlatformService = platformService; - DataAccessContext = dataAccessContext; - Engines = engines; - } - - protected IRepository Engines { get; } - protected IDistributedReaderWriterLockFactory LockFactory { get; } - protected IPlatformService PlatformService { get; } - protected IDataAccessContext DataAccessContext { get; } - - public abstract TranslationEngineType Type { get; } - - public virtual async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - CancellationToken cancellationToken = default - ) - { - await Engines.InsertAsync( - new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage - }, - cancellationToken - ); - } - - public virtual async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - await Engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await LockFactory.DeleteAsync(engineId, cancellationToken); - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - } - - public virtual async Task StartBuildAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await StartBuildInternalAsync(engineId, buildId, corpora, cancellationToken); - } - } - - public virtual async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildInternalAsync(engineId, cancellationToken); - } - } - - public virtual Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public virtual Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - protected abstract Expression> GetJobExpression( - string engineId, - string buildId, - IReadOnlyList corpora - ); - - protected async Task StartBuildInternalAsync( - string engineId, - string buildId, - IReadOnlyList corpora, - CancellationToken cancellationToken - ) - { - // If there is a pending job, then no need to start a new one. - if ( - await Engines.ExistsAsync( - e => - e.EngineId == engineId && (e.BuildState == BuildState.Pending || e.BuildState == BuildState.Active), - cancellationToken - ) - ) - throw new InvalidOperationException("Engine is already building or pending."); - - // Schedule the job to occur way in the future, just so we can get the job id. - string jobId = _jobClient.Schedule(GetJobExpression(engineId, buildId, corpora), TimeSpan.FromDays(10000)); - try - { - await Engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set(e => e.BuildState, BuildState.Pending) - .Set(e => e.IsCanceled, false) - .Set(e => e.JobId, jobId) - .Set(e => e.BuildId, buildId), - cancellationToken: CancellationToken.None - ); - // Enqueue the job now that the build has been created. - _jobClient.Requeue(jobId); - } - catch - { - _jobClient.Delete(jobId); - throw; - } - } - - protected async Task CancelBuildInternalAsync(string engineId, CancellationToken cancellationToken) - { - await DataAccessContext.BeginTransactionAsync(cancellationToken); - // First, try to cancel a job that hasn't started yet - TranslationEngine? engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Pending, - u => u.Set(b => b.BuildState, BuildState.None).Set(e => e.IsCanceled, true), - cancellationToken: cancellationToken - ); - bool notifyPlatform = false; - if (engine is not null) - { - notifyPlatform = true; - } - else - { - // Second, try to cancel a job that is already running - engine = await Engines.UpdateAsync( - e => e.EngineId == engineId && e.BuildState == BuildState.Active, - u => u.Set(b => b.IsCanceled, true), - cancellationToken: cancellationToken - ); - } - if (engine is not null) - { - // If pending, the job will be deleted from the queue, otherwise this will trigger the cancellation token - _jobClient.Delete(engine.JobId); - if (notifyPlatform) - await PlatformService.BuildCanceledAsync(engine.BuildId!, CancellationToken.None); - } - await DataAccessContext.CommitTransactionAsync(CancellationToken.None); - return engine?.BuildId; - } - - protected async Task WaitForBuildToFinishAsync( - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - using ISubscription sub = await Engines.SubscribeAsync( - e => e.EngineId == engineId && e.BuildId == buildId, - cancellationToken - ); - if (sub.Change.Entity is null) - return true; - - var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(20); - while (DateTime.UtcNow < timeout) - { - await sub.WaitForChangeAsync(TimeSpan.FromSeconds(2), cancellationToken); - TranslationEngine? engine = sub.Change.Entity; - if (engine is null || engine.BuildState is BuildState.None) - return true; - } - return false; - } - - protected async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException(""); - return engine; - } - - protected async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildState != BuildState.None || engine.BuildRevision == 0) //TranslationEngine.Revision - I don't see it used anywhere - throw new RpcException(new Status(StatusCode.FailedPrecondition, "The engine must be built first")); - return engine; - } -} diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs deleted file mode 100644 index 21868db71..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907084316.cs +++ /dev/null @@ -1,47 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.S3.Transfer; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs deleted file mode 100644 index 0b07fae03..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907131743.cs +++ /dev/null @@ -1,47 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.Runtime; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs deleted file mode 100644 index a808510e6..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141855.cs +++ /dev/null @@ -1,49 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.Runtime; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -using Nito.AsyncEx.Synchronous; - -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs deleted file mode 100644 index 981c59ef5..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141856.cs +++ /dev/null @@ -1,48 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.Runtime; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -using Nito.AsyncEx.Synchronous; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs deleted file mode 100644 index 8544d1e96..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141900.cs +++ /dev/null @@ -1,48 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.Runtime; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -global using Nito.AsyncEx.Synchronous; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs b/.history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs deleted file mode 100644 index 8544d1e96..000000000 --- a/.history/src/SIL.Machine.AspNetCore/Usings_20230907141901.cs +++ /dev/null @@ -1,48 +0,0 @@ -global using System.Collections; -global using System.Collections.Concurrent; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Amazon; -global using Amazon.S3; -global using Amazon.S3.Model; -global using Amazon.Runtime; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -global using Nito.AsyncEx.Synchronous; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.ObjectModel; -global using SIL.WritingSystems; diff --git a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs b/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs deleted file mode 100644 index f9d01a7d6..000000000 --- a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230906141540.cs +++ /dev/null @@ -1,192 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class ClearMLNmtEngineServiceTests -{ - [Test] - public async Task CancelBuildAsync() - { - using var env = new TestEnvironment(); - env.ClearMLService - .CreateTaskAsync( - Arg.Any(), - "project1", - "engine1", - "es", - "en", - "memory:///", - Arg.Any() - ) - .Returns(Task.FromResult("task1")); - var task = new ClearMLTask - { - Id = "task1", - Project = new ClearMLProject { Id = "project1" }, - Status = ClearMLTaskStatus.InProgress - }; - bool first = true; - env.ClearMLService - .GetTaskByNameAsync(Arg.Any(), Arg.Any()) - .Returns(x => - { - if (first) - { - first = false; - return Task.FromResult(null); - } - return Task.FromResult(task); - }); - env.ClearMLService - .GetTaskByIdAsync("task1", Arg.Any()) - .Returns(Task.FromResult(task)); - await env.Service.StartBuildAsync("engine1", "build1", Array.Empty()); - await env.WaitForBuildToStartAsync(); - TranslationEngine engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildState, Is.EqualTo(BuildState.Active)); - await env.Service.CancelBuildAsync("engine1"); - await env.WaitForBuildToFinishAsync(); - engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildState, Is.EqualTo(BuildState.None)); - await env.ClearMLService.Received().StopTaskAsync("task1", Arg.Any()); - } - - private class TestEnvironment : DisposableBase - { - private readonly MemoryStorage _memoryStorage; - private readonly BackgroundJobClient _jobClient; - private BackgroundJobServer _jobServer; - private readonly IDistributedReaderWriterLockFactory _lockFactory; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - - public TestEnvironment() - { - Engines = new MemoryRepository(); - Engines.Add( - new TranslationEngine - { - Id = "engine1", - EngineId = "engine1", - SourceLanguage = "es", - TargetLanguage = "en" - } - ); - EngineOptions = new SmtTransferEngineOptions(); - _memoryStorage = new MemoryStorage(); - _jobClient = new BackgroundJobClient(_memoryStorage); - PlatformService = Substitute.For(); - ClearMLService = Substitute.For(); - ClearMLService - .GetProjectIdAsync(Arg.Any(), Arg.Any()) - .Returns(Task.FromResult("project1")); - _lockFactory = new DistributedReaderWriterLockFactory( - new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), - new MemoryRepository(), - new ObjectIdGenerator() - ); - _sharedFileService = new SharedFileService(); - _options = Substitute.For>(); - _options.CurrentValue.Returns( - new ClearMLNmtEngineOptions { BuildPollingTimeout = TimeSpan.FromMilliseconds(50) } - ); - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - public ClearMLNmtEngineService Service { get; private set; } - public MemoryRepository Engines { get; } - public SmtTransferEngineOptions EngineOptions { get; } - public IPlatformService PlatformService { get; } - public IClearMLService ClearMLService { get; } - - public void StopServer() - { - _jobServer.Dispose(); - } - - public void StartServer() - { - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - private BackgroundJobServer CreateJobServer() - { - var jobServerOptions = new BackgroundJobServerOptions - { - Activator = new EnvActivator(this), - Queues = new[] { "nmt" }, - CancellationCheckInterval = TimeSpan.FromMilliseconds(100), - }; - return new BackgroundJobServer(jobServerOptions, _memoryStorage); - } - - private ClearMLNmtEngineService CreateService() - { - return new ClearMLNmtEngineService( - _jobClient, - PlatformService, - _lockFactory, - new MemoryDataAccessContext(), - Engines, - ClearMLService - ); - } - - public Task WaitForBuildToFinishAsync() - { - return WaitForBuildState(e => e.BuildState is BuildState.None); - } - - public Task WaitForBuildToStartAsync() - { - return WaitForBuildState(e => e.BuildState is BuildState.Active); - } - - private async Task WaitForBuildState(Func predicate) - { - using ISubscription subscription = await Engines.SubscribeAsync( - e => e.EngineId == "engine1" - ); - while (true) - { - TranslationEngine? build = subscription.Change.Entity; - if (build is not null && predicate(build)) - break; - await subscription.WaitForChangeAsync(); - } - } - - protected override void DisposeManagedResources() - { - _jobServer.Dispose(); - } - - private class EnvActivator : JobActivator - { - private readonly TestEnvironment _env; - - public EnvActivator(TestEnvironment env) - { - _env = env; - } - - public override object ActivateJob(Type jobType) - { - if (jobType == typeof(ClearMLNmtEngineBuildJob)) - { - return new ClearMLNmtEngineBuildJob( - _env.PlatformService, - _env.Engines, - Substitute.For>(), - _env.ClearMLService, - _env._sharedFileService, - _env._options, - Substitute.For() - ); - } - return base.ActivateJob(jobType); - } - } - } -} diff --git a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs b/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs deleted file mode 100644 index 614dabc52..000000000 --- a/.history/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLNmtEngineServiceTests_20230907185104.cs +++ /dev/null @@ -1,192 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class ClearMLNmtEngineServiceTests -{ - [Test] - public async Task CancelBuildAsync() - { - using var env = new TestEnvironment(); - env.ClearMLService - .CreateTaskAsync( - Arg.Any(), - "project1", - "engine1", - "es", - "en", - "memory:///", - Arg.Any() - ) - .Returns(Task.FromResult("task1")); - var task = new ClearMLTask - { - Id = "task1", - Project = new ClearMLProject { Id = "project1" }, - Status = ClearMLTaskStatus.InProgress - }; - bool first = true; - env.ClearMLService - .GetTaskByNameAsync(Arg.Any(), Arg.Any()) - .Returns(x => - { - if (first) - { - first = false; - return Task.FromResult(null); - } - return Task.FromResult(task); - }); - env.ClearMLService - .GetTaskByIdAsync("task1", Arg.Any()) - .Returns(Task.FromResult(task)); - await env.Service.StartBuildAsync("engine1", "build1", Array.Empty()); - await env.WaitForBuildToStartAsync(); - TranslationEngine engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildState, Is.EqualTo(BuildState.Active)); - await env.Service.CancelBuildAsync("engine1"); - await env.WaitForBuildToFinishAsync(); - engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildState, Is.EqualTo(BuildState.None)); - await env.ClearMLService.Received().StopTaskAsync("task1", Arg.Any()); - } - - private class TestEnvironment : DisposableBase - { - private readonly MemoryStorage _memoryStorage; - private readonly BackgroundJobClient _jobClient; - private BackgroundJobServer _jobServer; - private readonly IDistributedReaderWriterLockFactory _lockFactory; - private readonly ISharedFileService _sharedFileService; - private readonly IOptionsMonitor _options; - - public TestEnvironment() - { - Engines = new MemoryRepository(); - Engines.Add( - new TranslationEngine - { - Id = "engine1", - EngineId = "engine1", - SourceLanguage = "es", - TargetLanguage = "en" - } - ); - EngineOptions = new SmtTransferEngineOptions(); - _memoryStorage = new MemoryStorage(); - _jobClient = new BackgroundJobClient(_memoryStorage); - PlatformService = Substitute.For(); - ClearMLService = Substitute.For(); - ClearMLService - .GetProjectIdAsync(Arg.Any(), Arg.Any()) - .Returns(Task.FromResult("project1")); - _lockFactory = new DistributedReaderWriterLockFactory( - new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), - new MemoryRepository(), - new ObjectIdGenerator() - ); - _sharedFileService = new SharedFileService(Substitute.For()); - _options = Substitute.For>(); - _options.CurrentValue.Returns( - new ClearMLNmtEngineOptions { BuildPollingTimeout = TimeSpan.FromMilliseconds(50) } - ); - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - public ClearMLNmtEngineService Service { get; private set; } - public MemoryRepository Engines { get; } - public SmtTransferEngineOptions EngineOptions { get; } - public IPlatformService PlatformService { get; } - public IClearMLService ClearMLService { get; } - - public void StopServer() - { - _jobServer.Dispose(); - } - - public void StartServer() - { - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - private BackgroundJobServer CreateJobServer() - { - var jobServerOptions = new BackgroundJobServerOptions - { - Activator = new EnvActivator(this), - Queues = new[] { "nmt" }, - CancellationCheckInterval = TimeSpan.FromMilliseconds(100), - }; - return new BackgroundJobServer(jobServerOptions, _memoryStorage); - } - - private ClearMLNmtEngineService CreateService() - { - return new ClearMLNmtEngineService( - _jobClient, - PlatformService, - _lockFactory, - new MemoryDataAccessContext(), - Engines, - ClearMLService - ); - } - - public Task WaitForBuildToFinishAsync() - { - return WaitForBuildState(e => e.BuildState is BuildState.None); - } - - public Task WaitForBuildToStartAsync() - { - return WaitForBuildState(e => e.BuildState is BuildState.Active); - } - - private async Task WaitForBuildState(Func predicate) - { - using ISubscription subscription = await Engines.SubscribeAsync( - e => e.EngineId == "engine1" - ); - while (true) - { - TranslationEngine? build = subscription.Change.Entity; - if (build is not null && predicate(build)) - break; - await subscription.WaitForChangeAsync(); - } - } - - protected override void DisposeManagedResources() - { - _jobServer.Dispose(); - } - - private class EnvActivator : JobActivator - { - private readonly TestEnvironment _env; - - public EnvActivator(TestEnvironment env) - { - _env = env; - } - - public override object ActivateJob(Type jobType) - { - if (jobType == typeof(ClearMLNmtEngineBuildJob)) - { - return new ClearMLNmtEngineBuildJob( - _env.PlatformService, - _env.Engines, - Substitute.For>(), - _env.ClearMLService, - _env._sharedFileService, - _env._options, - Substitute.For() - ); - } - return base.ActivateJob(jobType); - } - } - } -}