From 697968060471dd884db1dbd27b153c2637970625 Mon Sep 17 00:00:00 2001 From: Damien Daspit Date: Wed, 10 Jul 2024 12:13:51 -0500 Subject: [PATCH] Remove Serval engine (#219) --- .github/workflows/ci-e2e.yml | 74 - .github/workflows/ci.yml | 1 - .github/workflows/docker-build-push.yml | 40 - Machine.sln | 28 - dockerfile | 22 - dockerfile.development | 4 - samples/corpora.ipynb | 675 - samples/data/VBL-PT/631JNspavbl.SFM | 138 - samples/data/VBL-PT/642JNspavbl.SFM | 24 - samples/data/VBL-PT/653JNspavbl.SFM | 25 - samples/data/VBL-PT/BookNames.xml | 126 - samples/data/VBL-PT/Settings.xml | 71 - samples/data/VBL-PT/custom.vrs | 22 - samples/data/VBL-PT/es-Latn.ldml | 877 -- samples/data/WEB-DBL/license.xml | 1 - samples/data/WEB-DBL/metadata.xml | 139 - samples/data/WEB-DBL/release/USX_1/1JN.usx | 107 - samples/data/WEB-DBL/release/USX_1/2JN.usx | 24 - samples/data/WEB-DBL/release/USX_1/3JN.usx | 31 - samples/data/WEB-DBL/release/eng_en.ldml | 1 - samples/data/WEB-DBL/release/styles.xml | 1817 --- .../data/WEB-DBL/release/versification.vrs | 395 - samples/data/WEB-PT/631JNengWEB14.SFM | 199 - samples/data/WEB-PT/642JNengWEB14.SFM | 25 - samples/data/WEB-PT/653JNengWEB14.SFM | 29 - samples/data/WEB-PT/BookNames.xml | 126 - samples/data/WEB-PT/Settings.xml | 86 - samples/data/WEB-PT/custom.sty | 353 - samples/data/WEB-PT/custom.vrs | 30 - samples/data/WEB-PT/en.ldml | 1 - samples/data/WEB-PT/frtbak_old.sty | 2278 ---- samples/data/en-hc.xml | 265 - samples/data/en.txt | 1000 -- samples/data/en_tok.txt | 1000 -- samples/data/smt.cfg | 29 - samples/data/sp-hc.xml | 282 - samples/data/sp.txt | 1000 -- samples/machine_translation.ipynb | 467 - samples/tokenization.ipynb | 419 - samples/word_alignment.ipynb | 435 - .../Configuration/BuildJobOptions.cs | 8 - .../Configuration/ClearMLBuildQueue.cs | 9 - .../Configuration/ClearMLOptions.cs | 13 - .../IEndpointRouteBuilderExtensions.cs | 11 - .../Configuration/IMachineBuilder.cs | 7 - .../IMachineBuilderExtensions.cs | 437 - .../IServiceCollectionExtensions.cs | 54 - .../Configuration/MachineBuilder.cs | 7 - .../Configuration/MessageOutboxOptions.cs | 9 - .../Configuration/ServiceOptions.cs | 8 - .../Configuration/SharedFileOptions.cs | 11 - .../Configuration/SmtTransferEngineOptions.cs | 10 - .../Configuration/ThotSmtModelOptions.cs | 14 - src/SIL.Machine.AspNetCore/Models/Build.cs | 32 - .../Models/ClearMLMetricsEvent.cs | 12 - .../Models/ClearMLProject.cs | 6 - .../Models/ClearMLTask.cs | 35 - src/SIL.Machine.AspNetCore/Models/Corpus.cs | 14 - .../Models/CorpusFile.cs | 14 - src/SIL.Machine.AspNetCore/Models/Lock.cs | 8 - .../Models/ModelDownloadUrl.cs | 8 - src/SIL.Machine.AspNetCore/Models/Outbox.cs | 10 - .../Models/OutboxMessage.cs | 15 - .../Models/Pretranslation.cs | 9 - src/SIL.Machine.AspNetCore/Models/RWLock.cs | 25 - .../Models/TrainSegmentPair.cs | 11 - .../Models/TranslationEngine.cs | 14 - .../Properties/AssemblyInfo.cs | 2 - .../SIL.Machine.AspNetCore.csproj | 61 - .../Services/BuildJobService.cs | 211 - .../Services/BuildProgress.cs | 25 - .../Services/CancellationInterceptor.cs | 30 - .../Services/ClearMLAuthenticationService.cs | 73 - .../Services/ClearMLBuildJobRunner.cs | 88 - .../Services/ClearMLHealthCheck.cs | 100 - .../Services/ClearMLMonitorService.cs | 407 - .../Services/ClearMLService.cs | 221 - .../Services/CorpusService.cs | 51 - .../Services/DistributedReaderWriterLock.cs | 177 - .../DistributedReaderWriterLockFactory.cs | 77 - .../Services/FileSystem.cs | 25 - .../Services/HangfireBuildJob.cs | 180 - .../Services/HangfireBuildJobRunner.cs | 85 - .../Services/HangfireHealthCheck.cs | 25 - .../Services/IBuildJobRunner.cs | 29 - .../Services/IBuildJobService.cs | 41 - .../Services/IClearMLAuthenticationService.cs | 6 - .../Services/IClearMLBuildJobFactory.cs | 16 - .../Services/IClearMLQueueService.cs | 6 - .../Services/IClearMLService.cs | 30 - .../Services/ICorpusService.cs | 7 - .../Services/IDistributedReaderWriterLock.cs | 7 - .../IDistributedReaderWriterLockFactory.cs | 8 - .../Services/IFileStorage.cs | 20 - .../Services/IFileSystem.cs | 9 - .../Services/IHangfireBuildJobFactory.cs | 8 - .../Services/ILanguageTagService.cs | 6 - .../Services/IMessageOutboxService.cs | 13 - .../Services/IOutboxMessageHandler.cs | 13 - .../Services/IPlatformService.cs | 30 - .../Services/ISharedFileService.cs | 24 - .../Services/ISmtModelFactory.cs | 22 - .../Services/ITransferEngineFactory.cs | 14 - .../Services/ITranslationEngineService.cs | 49 - .../Services/ITruecaserFactory.cs | 13 - .../Services/InMemoryStorage.cs | 142 - .../Services/LanguageTagService.cs | 170 - .../Services/LocalStorage.cs | 78 - .../Services/MessageOutboxDeliveryService.cs | 174 - .../Services/MessageOutboxService.cs | 75 - .../Services/ModelCleanupService.cs | 59 - .../Services/NmtClearMLBuildJobFactory.cs | 58 - .../Services/NmtEngineService.cs | 204 - .../Services/NmtHangfireBuildJobFactory.cs | 26 - .../Services/NmtPreprocessBuildJob.cs | 31 - .../Services/PostprocessBuildJob.cs | 88 - .../Services/PreprocessBuildJob.cs | 440 - .../Services/S3FileStorage.cs | 125 - .../Services/S3HealthCheck.cs | 67 - .../Services/S3WriteStream.cs | 222 - .../Services/ServalPlatformOutboxConstants.cs | 14 - .../ServalPlatformOutboxMessageHandler.cs | 92 - .../Services/ServalPlatformService.cs | 140 - .../ServalTranslationEngineServiceV1.cs | 341 - .../Services/SharedFileService.cs | 104 - .../Services/SmtTransferBuildJob.cs | 157 - .../SmtTransferClearMLBuildJobFactory.cs | 49 - .../SmtTransferEngineCommitService.cs | 38 - .../Services/SmtTransferEngineService.cs | 272 - .../Services/SmtTransferEngineState.cs | 128 - .../Services/SmtTransferEngineStateService.cs | 72 - .../SmtTransferHangfireBuildJobFactory.cs | 33 - .../SmtTransferPostprocessBuildJob.cs | 74 - .../Services/SmtTransferTrainBuildJob.cs | 237 - .../Services/ThotSmtModelFactory.cs | 110 - .../Services/TransferEngineFactory.cs | 61 - .../Services/TranslationEngineType.cs | 7 - .../Services/UnigramTruecaserFactory.cs | 37 - .../Services/UnimplementedInterceptor.cs | 22 - src/SIL.Machine.AspNetCore/Usings.cs | 59 - .../Utils/AsyncDisposableBase.cs | 19 - .../Utils/AsyncTimer.cs | 69 - .../Utils/CustomEnumConverterFactory.cs | 155 - .../Utils/DictionaryStringConverter.cs | 73 - .../Utils/EngineNotBuiltException.cs | 4 - .../Utils/RecurrentTask.cs | 40 - .../Utils/SharedFileUtils.cs | 28 - .../Utils/StartupTask.cs | 19 - .../data/.gitattributes | 5 - .../data/flores200languages.csv | 205 - .../data/thot-new-model/lm/trg.lm | 6 - .../data/thot-new-model/lm/trg.lm.weights | 1 - .../data/thot-new-model/lm/trg.lm.wp | 1 - .../data/thot-new-model/smt.cfg | 29 - .../data/thot-new-model/tm/src_trg.lambda | 1 - .../thot-new-model/tm/src_trg.seglentable | 10201 ---------------- .../thot-new-model/tm/src_trg.srcsegmlentable | 1 - .../thot-new-model/tm/src_trg.trgsegmlentable | 1 - .../data/thot-new-model/tm/src_trg.ttable | 0 .../thot-new-model/tm/src_trg_invswm.anji | 0 .../tm/src_trg_invswm.anjm1ip_anji | 0 .../tm/src_trg_invswm.asifactor | 1 - .../tm/src_trg_invswm.hmm_alignd | Bin 22 -> 0 bytes .../tm/src_trg_invswm.hmm_lexnd | Bin 32 -> 0 bytes .../thot-new-model/tm/src_trg_invswm.hmm_p0 | 1 - .../tm/src_trg_invswm.lsifactor | 1 - .../thot-new-model/tm/src_trg_invswm.msinfo | 2 - .../thot-new-model/tm/src_trg_invswm.slmodel | 3 - .../data/thot-new-model/tm/src_trg_invswm.src | 1 - .../thot-new-model/tm/src_trg_invswm.srctrgc | 1 - .../thot-new-model/tm/src_trg_invswm.svcb | 4 - .../data/thot-new-model/tm/src_trg_invswm.trg | 1 - .../thot-new-model/tm/src_trg_invswm.tvcb | 4 - .../data/thot-new-model/tm/src_trg_swm.anji | 0 .../tm/src_trg_swm.anjm1ip_anji | 0 .../thot-new-model/tm/src_trg_swm.asifactor | 1 - .../thot-new-model/tm/src_trg_swm.hmm_alignd | Bin 22 -> 0 bytes .../thot-new-model/tm/src_trg_swm.hmm_lexnd | Bin 32 -> 0 bytes .../data/thot-new-model/tm/src_trg_swm.hmm_p0 | 1 - .../thot-new-model/tm/src_trg_swm.lsifactor | 1 - .../data/thot-new-model/tm/src_trg_swm.msinfo | 2 - .../thot-new-model/tm/src_trg_swm.slmodel | 3 - .../data/thot-new-model/tm/src_trg_swm.src | 1 - .../thot-new-model/tm/src_trg_swm.srctrgc | 1 - .../data/thot-new-model/tm/src_trg_swm.svcb | 4 - .../data/thot-new-model/tm/src_trg_swm.trg | 1 - .../data/thot-new-model/tm/src_trg_swm.tvcb | 4 - .../Program.cs | 39 - .../Properties/launchSettings.json | 11 - .../SIL.Machine.Serval.EngineServer.csproj | 36 - .../appsettings.Development.json | 21 - .../appsettings.Production.json | 8 - .../appsettings.Staging.json | 8 - .../appsettings.json | 43 - src/SIL.Machine.Serval.JobServer/Program.cs | 30 - .../Properties/launchSettings.json | 12 - .../SIL.Machine.Serval.JobServer.csproj | 37 - .../appsettings.Development.json | 21 - .../appsettings.Production.json | 8 - .../appsettings.Staging.json | 8 - .../appsettings.json | 43 - .../SIL.Machine.AspNetCore.Tests.csproj | 52 - .../Services/ClearMLServiceTests.cs | 50 - ...DistributedReaderWriterLockFactoryTests.cs | 81 - .../DistributedReaderWriterLockTests.cs | 405 - .../Services/InMemoryStorageTests.cs | 91 - .../Services/LanguageTagServiceTests.cs | 69 - .../Services/LocalStorageTests.cs | 96 - .../MessageOutboxDeliveryServiceTests.cs | 221 - .../Services/MessageOutboxServiceTests.cs | 93 - .../Services/ModelCleanupServiceTests.cs | 104 - .../NmtClearMLBuildJobFactoryTests.cs | 123 - .../Services/NmtEngineServiceTests.cs | 324 - .../Services/PreprocessBuildJobTests.cs | 581 - ...ServalPlatformOutboxMessageHandlerTests.cs | 114 - .../Services/SmtTransferEngineServiceTests.cs | 742 -- .../Services/data/pt-source1/04LEVTe1.SFM | 8 - .../Services/data/pt-source1/131CHTe1.SFM | 12 - .../Services/data/pt-source1/41MATTe1.SFM | 16 - .../Services/data/pt-source1/42MRKTe1.SFM | 4 - .../Services/data/pt-source1/Settings.xml | 34 - .../data/pt-source1/TermRenderings.xml | 9 - .../Services/data/pt-source1/custom.vrs | 31 - .../Services/data/pt-source2/04LEVTe3.SFM | 7 - .../Services/data/pt-source2/131CHTe3.SFM | 16 - .../Services/data/pt-source2/41MATTe3.SFM | 19 - .../Services/data/pt-source2/42MRKTe3.SFM | 7 - .../Services/data/pt-source2/Settings.xml | 34 - .../data/pt-source2/TermRenderings.xml | 9 - .../Services/data/pt-source2/custom.vrs | 31 - .../Services/data/pt-target1/41MATTe2.SFM | 17 - .../Services/data/pt-target1/42MRKTe2.SFM | 4 - .../Services/data/pt-target1/Settings.xml | 33 - .../data/pt-target1/TermRenderings.xml | 9 - .../Services/data/pt-target1/custom.vrs | 31 - .../Services/data/source1.txt | 7 - .../Services/data/source2.txt | 7 - .../Services/data/target1.txt | 7 - tests/SIL.Machine.AspNetCore.Tests/Usings.cs | 27 - 239 files changed, 34438 deletions(-) delete mode 100644 .github/workflows/ci-e2e.yml delete mode 100644 .github/workflows/docker-build-push.yml delete mode 100644 dockerfile delete mode 100644 dockerfile.development delete mode 100644 samples/corpora.ipynb delete mode 100644 samples/data/VBL-PT/631JNspavbl.SFM delete mode 100644 samples/data/VBL-PT/642JNspavbl.SFM delete mode 100644 samples/data/VBL-PT/653JNspavbl.SFM delete mode 100644 samples/data/VBL-PT/BookNames.xml delete mode 100644 samples/data/VBL-PT/Settings.xml delete mode 100644 samples/data/VBL-PT/custom.vrs delete mode 100644 samples/data/VBL-PT/es-Latn.ldml delete mode 100644 samples/data/WEB-DBL/license.xml delete mode 100644 samples/data/WEB-DBL/metadata.xml delete mode 100644 samples/data/WEB-DBL/release/USX_1/1JN.usx delete mode 100644 samples/data/WEB-DBL/release/USX_1/2JN.usx delete mode 100644 samples/data/WEB-DBL/release/USX_1/3JN.usx delete mode 100644 samples/data/WEB-DBL/release/eng_en.ldml delete mode 100644 samples/data/WEB-DBL/release/styles.xml delete mode 100644 samples/data/WEB-DBL/release/versification.vrs delete mode 100644 samples/data/WEB-PT/631JNengWEB14.SFM delete mode 100644 samples/data/WEB-PT/642JNengWEB14.SFM delete mode 100644 samples/data/WEB-PT/653JNengWEB14.SFM delete mode 100644 samples/data/WEB-PT/BookNames.xml delete mode 100644 samples/data/WEB-PT/Settings.xml delete mode 100644 samples/data/WEB-PT/custom.sty delete mode 100644 samples/data/WEB-PT/custom.vrs delete mode 100644 samples/data/WEB-PT/en.ldml delete mode 100644 samples/data/WEB-PT/frtbak_old.sty delete mode 100644 samples/data/en-hc.xml delete mode 100644 samples/data/en.txt delete mode 100644 samples/data/en_tok.txt delete mode 100644 samples/data/smt.cfg delete mode 100644 samples/data/sp-hc.xml delete mode 100644 samples/data/sp.txt delete mode 100644 samples/machine_translation.ipynb delete mode 100644 samples/tokenization.ipynb delete mode 100644 samples/word_alignment.ipynb delete mode 100644 src/SIL.Machine.AspNetCore/Configuration/BuildJobOptions.cs delete mode 100644 src/SIL.Machine.AspNetCore/Configuration/ClearMLBuildQueue.cs delete mode 100644 src/SIL.Machine.AspNetCore/Configuration/ClearMLOptions.cs delete mode 100644 src/SIL.Machine.AspNetCore/Configuration/IEndpointRouteBuilderExtensions.cs delete mode 100644 src/SIL.Machine.AspNetCore/Configuration/IMachineBuilder.cs delete mode 100644 src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs delete mode 100644 src/SIL.Machine.AspNetCore/Configuration/IServiceCollectionExtensions.cs delete mode 100644 src/SIL.Machine.AspNetCore/Configuration/MachineBuilder.cs delete mode 100644 src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs delete mode 100644 src/SIL.Machine.AspNetCore/Configuration/ServiceOptions.cs delete mode 100644 src/SIL.Machine.AspNetCore/Configuration/SharedFileOptions.cs delete mode 100644 src/SIL.Machine.AspNetCore/Configuration/SmtTransferEngineOptions.cs delete mode 100644 src/SIL.Machine.AspNetCore/Configuration/ThotSmtModelOptions.cs delete mode 100644 src/SIL.Machine.AspNetCore/Models/Build.cs delete mode 100644 src/SIL.Machine.AspNetCore/Models/ClearMLMetricsEvent.cs delete mode 100644 src/SIL.Machine.AspNetCore/Models/ClearMLProject.cs delete mode 100644 src/SIL.Machine.AspNetCore/Models/ClearMLTask.cs delete mode 100644 src/SIL.Machine.AspNetCore/Models/Corpus.cs delete mode 100644 src/SIL.Machine.AspNetCore/Models/CorpusFile.cs delete mode 100644 src/SIL.Machine.AspNetCore/Models/Lock.cs delete mode 100644 src/SIL.Machine.AspNetCore/Models/ModelDownloadUrl.cs delete mode 100644 src/SIL.Machine.AspNetCore/Models/Outbox.cs delete mode 100644 src/SIL.Machine.AspNetCore/Models/OutboxMessage.cs delete mode 100644 src/SIL.Machine.AspNetCore/Models/Pretranslation.cs delete mode 100644 src/SIL.Machine.AspNetCore/Models/RWLock.cs delete mode 100644 src/SIL.Machine.AspNetCore/Models/TrainSegmentPair.cs delete mode 100644 src/SIL.Machine.AspNetCore/Models/TranslationEngine.cs delete mode 100644 src/SIL.Machine.AspNetCore/Properties/AssemblyInfo.cs delete mode 100644 src/SIL.Machine.AspNetCore/SIL.Machine.AspNetCore.csproj delete mode 100644 src/SIL.Machine.AspNetCore/Services/BuildJobService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/BuildProgress.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/CancellationInterceptor.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ClearMLAuthenticationService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ClearMLBuildJobRunner.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ClearMLHealthCheck.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ClearMLMonitorService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ClearMLService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/CorpusService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/DistributedReaderWriterLock.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/DistributedReaderWriterLockFactory.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/FileSystem.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/HangfireBuildJob.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/HangfireBuildJobRunner.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/HangfireHealthCheck.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/IBuildJobRunner.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/IBuildJobService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/IClearMLAuthenticationService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/IClearMLBuildJobFactory.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/IClearMLQueueService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/IClearMLService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ICorpusService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/IDistributedReaderWriterLock.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/IDistributedReaderWriterLockFactory.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/IFileStorage.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/IFileSystem.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/IHangfireBuildJobFactory.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ILanguageTagService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/IMessageOutboxService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/IOutboxMessageHandler.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/IPlatformService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ISharedFileService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ISmtModelFactory.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ITransferEngineFactory.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ITranslationEngineService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ITruecaserFactory.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/InMemoryStorage.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/LanguageTagService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/LocalStorage.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ModelCleanupService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/NmtClearMLBuildJobFactory.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/NmtEngineService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/NmtHangfireBuildJobFactory.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/NmtPreprocessBuildJob.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/PreprocessBuildJob.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/S3FileStorage.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/S3HealthCheck.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/S3WriteStream.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxConstants.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxMessageHandler.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ServalPlatformService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ServalTranslationEngineServiceV1.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/SharedFileService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/SmtTransferBuildJob.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/SmtTransferClearMLBuildJobFactory.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/SmtTransferEngineCommitService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/SmtTransferEngineState.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/SmtTransferEngineStateService.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/SmtTransferHangfireBuildJobFactory.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/SmtTransferPostprocessBuildJob.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/SmtTransferTrainBuildJob.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/ThotSmtModelFactory.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/TransferEngineFactory.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/TranslationEngineType.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/UnigramTruecaserFactory.cs delete mode 100644 src/SIL.Machine.AspNetCore/Services/UnimplementedInterceptor.cs delete mode 100644 src/SIL.Machine.AspNetCore/Usings.cs delete mode 100644 src/SIL.Machine.AspNetCore/Utils/AsyncDisposableBase.cs delete mode 100644 src/SIL.Machine.AspNetCore/Utils/AsyncTimer.cs delete mode 100644 src/SIL.Machine.AspNetCore/Utils/CustomEnumConverterFactory.cs delete mode 100644 src/SIL.Machine.AspNetCore/Utils/DictionaryStringConverter.cs delete mode 100644 src/SIL.Machine.AspNetCore/Utils/EngineNotBuiltException.cs delete mode 100644 src/SIL.Machine.AspNetCore/Utils/RecurrentTask.cs delete mode 100644 src/SIL.Machine.AspNetCore/Utils/SharedFileUtils.cs delete mode 100644 src/SIL.Machine.AspNetCore/Utils/StartupTask.cs delete mode 100644 src/SIL.Machine.AspNetCore/data/.gitattributes delete mode 100644 src/SIL.Machine.AspNetCore/data/flores200languages.csv delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/lm/trg.lm delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/lm/trg.lm.weights delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/lm/trg.lm.wp delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/smt.cfg delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.lambda delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.seglentable delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.srcsegmlentable delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.trgsegmlentable delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.ttable delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.anji delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.anjm1ip_anji delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.asifactor delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.hmm_alignd delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.hmm_lexnd delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.hmm_p0 delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.lsifactor delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.msinfo delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.slmodel delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.src delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.srctrgc delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.svcb delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.trg delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.tvcb delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.anji delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.anjm1ip_anji delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.asifactor delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.hmm_alignd delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.hmm_lexnd delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.hmm_p0 delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.lsifactor delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.msinfo delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.slmodel delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.src delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.srctrgc delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.svcb delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.trg delete mode 100644 src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.tvcb delete mode 100644 src/SIL.Machine.Serval.EngineServer/Program.cs delete mode 100644 src/SIL.Machine.Serval.EngineServer/Properties/launchSettings.json delete mode 100644 src/SIL.Machine.Serval.EngineServer/SIL.Machine.Serval.EngineServer.csproj delete mode 100644 src/SIL.Machine.Serval.EngineServer/appsettings.Development.json delete mode 100644 src/SIL.Machine.Serval.EngineServer/appsettings.Production.json delete mode 100644 src/SIL.Machine.Serval.EngineServer/appsettings.Staging.json delete mode 100644 src/SIL.Machine.Serval.EngineServer/appsettings.json delete mode 100644 src/SIL.Machine.Serval.JobServer/Program.cs delete mode 100644 src/SIL.Machine.Serval.JobServer/Properties/launchSettings.json delete mode 100644 src/SIL.Machine.Serval.JobServer/SIL.Machine.Serval.JobServer.csproj delete mode 100644 src/SIL.Machine.Serval.JobServer/appsettings.Development.json delete mode 100644 src/SIL.Machine.Serval.JobServer/appsettings.Production.json delete mode 100644 src/SIL.Machine.Serval.JobServer/appsettings.Staging.json delete mode 100644 src/SIL.Machine.Serval.JobServer/appsettings.json delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/SIL.Machine.AspNetCore.Tests.csproj delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLServiceTests.cs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/DistributedReaderWriterLockFactoryTests.cs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/DistributedReaderWriterLockTests.cs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/InMemoryStorageTests.cs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/LanguageTagServiceTests.cs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/LocalStorageTests.cs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxServiceTests.cs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/ModelCleanupServiceTests.cs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/NmtClearMLBuildJobFactoryTests.cs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/NmtEngineServiceTests.cs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/PreprocessBuildJobTests.cs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/SmtTransferEngineServiceTests.cs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/04LEVTe1.SFM delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/131CHTe1.SFM delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/41MATTe1.SFM delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/42MRKTe1.SFM delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/Settings.xml delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/TermRenderings.xml delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/custom.vrs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/04LEVTe3.SFM delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/131CHTe3.SFM delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/41MATTe3.SFM delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/42MRKTe3.SFM delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/Settings.xml delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/TermRenderings.xml delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/custom.vrs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/41MATTe2.SFM delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/42MRKTe2.SFM delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/Settings.xml delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/TermRenderings.xml delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/custom.vrs delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/source1.txt delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/source2.txt delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Services/data/target1.txt delete mode 100644 tests/SIL.Machine.AspNetCore.Tests/Usings.cs diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml deleted file mode 100644 index 1581d083a..000000000 --- a/.github/workflows/ci-e2e.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: "CI Build: E2E tests" - -on: - push: - tags: - - 'docker_*' - workflow_dispatch: - -jobs: - build: - name: Build - runs-on: ubuntu-latest - timeout-minutes: 45 - - env: - SERVAL_CLIENT_ID: ${{ secrets.SERVAL_CLIENT_ID }} - SERVAL_CLIENT_SECRET: ${{ secrets.SERVAL_CLIENT_SECRET }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - ClearML_AccessKey: ${{ secrets.CLEARML_ACCESSKEY }} - ClearML_SecretKey: ${{ secrets.CLEARML_SECRETKEY }} - SERVAL_HOST_URL: http://localhost - SERVAL_AUTH_URL: https://sil-appbuilder.auth0.com - - steps: - - uses: actions/checkout@v3 - - # get version of machine.py - MACHINE_PY_IMAGE will force the docker compose to use the proper version of machine.py - - name: Install regctl - uses: iarekylew00t/regctl-installer@v1 - - - name: Getr Version of Machine.py - run: echo "MACHINE_PY_IMAGE=ghcr.io/sillsdev/machine.py:$(regctl image config ghcr.io/sillsdev/machine.py | jq -r ".config.Labels[\"org.opencontainers.image.version\"]")" >> $GITHUB_ENV - - - name: Setup .NET - uses: actions/setup-dotnet@v3 - with: - dotnet-version: 6.0.x - - - name: Check if there is a corresponding branch in Serval - id: find_matching_branch - continue-on-error: true - if: ${{ github.ref_name }} != "master" - run: cd .. && git clone https://github.com/sillsdev/serval.git --branch ${{ github.ref_name }} && dotnet build machine && dotnet build serval && docker compose -f "serval/docker-compose.yml" up -d && sleep 20 - - - name: Get Serval and start containers - if: steps.find_matching_branch.outcome != 'success' - run: dotnet build && cd .. && mkdir tmp && cd tmp && git clone https://github.com/sillsdev/serval.git && cd serval && dotnet build && export MACHINE_TESTING_DIR="../../machine" && docker compose -f "docker-compose.yml" up -d && sleep 20 #separate repos; allow time for mongo to start up properly - - - name: Debug network - run: docker ps -a && docker logs --since 10m serval_cntr && docker logs --since 10m echo_cntr && docker logs --since 10m machine-engine-cntr && docker logs --since 10m serval-mongo-1 && docker logs --since 10m machine-job-cntr - - - name: Pre-Test - run: sudo mkdir -p /var/lib/serval && sudo chmod 777 /var/lib/serval - - - name: Test - if: steps.find_matching_branch.outcome != 'success' - run: cd ../tmp/serval && dotnet test --no-build --verbosity normal --filter "TestCategory!=slow&TestCategory=E2E" - - - name: Test - if: steps.find_matching_branch.outcome == 'success' - run: cd ../serval && dotnet test --no-build --verbosity normal --filter "TestCategory!=slow&TestCategory=E2E" - - - name: Debug network (Post test) - if: ${{ failure() }} - run: docker ps -a && docker logs --since 10m serval_cntr && docker logs --since 10m echo_cntr && docker logs --since 10m machine-engine-cntr && docker logs --since 10m serval-mongo-1 && docker logs --since 10m machine-job-cntr - - - name: Stop containers main - if: ${{ (success() || failure()) && (steps.find_matching_branch.outcome != 'success')}} - run: docker compose -f "../tmp/serval/docker-compose.yml" down - - - name: Stop containers branch - if: ${{ (success() || failure()) && (steps.find_matching_branch.outcome == 'success')}} - run: docker compose -f "../serval/docker-compose.yml" down diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afb185e3a..1af216169 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,7 +97,6 @@ jobs: run: | dotnet pack src\SIL.Machine\SIL.Machine.csproj -c Release -o artifacts dotnet pack src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj -c Release -o artifacts - dotnet pack src\SIL.Machine.AspNetCore\SIL.Machine.AspNetCore.csproj -c Release -o artifacts dotnet pack src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj -c Release -o artifacts dotnet pack src\SIL.Machine.Tool\SIL.Machine.Tool.csproj -c Release -o artifacts dotnet pack src\SIL.Machine.Morphology.HermitCrab.Tool\SIL.Machine.Morphology.HermitCrab.Tool.csproj -c Release -o artifacts diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml deleted file mode 100644 index d05ffd3d8..000000000 --- a/.github/workflows/docker-build-push.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: docker build - -on: - push: - tags: - - 'docker_*' - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Generate Docker metadata - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ghcr.io/${{ github.repository }} - tags: | - type=match,pattern=docker_(.*),group=1 - flavor: | - latest=true - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/Machine.sln b/Machine.sln index 5d607ab30..019f56ad1 100644 --- a/Machine.sln +++ b/Machine.sln @@ -45,14 +45,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.Machine.Tokenization.Se EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.Machine.Tokenization.SentencePiece.Tests", "tests\SIL.Machine.Tokenization.SentencePiece.Tests\SIL.Machine.Tokenization.SentencePiece.Tests.csproj", "{AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.Machine.AspNetCore", "src\SIL.Machine.AspNetCore\SIL.Machine.AspNetCore.csproj", "{A04C171E-A8E5-40D2-AECA-9FDA6F5FE92B}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.Machine.Serval.EngineServer", "src\SIL.Machine.Serval.EngineServer\SIL.Machine.Serval.EngineServer.csproj", "{183B5AB9-1BD1-4000-97E7-90706D45A668}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.Machine.Serval.JobServer", "src\SIL.Machine.Serval.JobServer\SIL.Machine.Serval.JobServer.csproj", "{81CABDF5-7A9C-44A8-9A0A-2B17DED577A9}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SIL.Machine.AspNetCore.Tests", "tests\SIL.Machine.AspNetCore.Tests\SIL.Machine.AspNetCore.Tests.csproj", "{B5B83426-7099-4EAD-9E07-8D8CA61BABAB}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -107,22 +99,6 @@ Global {AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}.Debug|Any CPU.Build.0 = Debug|Any CPU {AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}.Release|Any CPU.ActiveCfg = Release|Any CPU {AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC}.Release|Any CPU.Build.0 = Release|Any CPU - {A04C171E-A8E5-40D2-AECA-9FDA6F5FE92B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A04C171E-A8E5-40D2-AECA-9FDA6F5FE92B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A04C171E-A8E5-40D2-AECA-9FDA6F5FE92B}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A04C171E-A8E5-40D2-AECA-9FDA6F5FE92B}.Release|Any CPU.Build.0 = Release|Any CPU - {183B5AB9-1BD1-4000-97E7-90706D45A668}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {183B5AB9-1BD1-4000-97E7-90706D45A668}.Debug|Any CPU.Build.0 = Debug|Any CPU - {183B5AB9-1BD1-4000-97E7-90706D45A668}.Release|Any CPU.ActiveCfg = Release|Any CPU - {183B5AB9-1BD1-4000-97E7-90706D45A668}.Release|Any CPU.Build.0 = Release|Any CPU - {81CABDF5-7A9C-44A8-9A0A-2B17DED577A9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {81CABDF5-7A9C-44A8-9A0A-2B17DED577A9}.Debug|Any CPU.Build.0 = Debug|Any CPU - {81CABDF5-7A9C-44A8-9A0A-2B17DED577A9}.Release|Any CPU.ActiveCfg = Release|Any CPU - {81CABDF5-7A9C-44A8-9A0A-2B17DED577A9}.Release|Any CPU.Build.0 = Release|Any CPU - {B5B83426-7099-4EAD-9E07-8D8CA61BABAB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B5B83426-7099-4EAD-9E07-8D8CA61BABAB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B5B83426-7099-4EAD-9E07-8D8CA61BABAB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B5B83426-7099-4EAD-9E07-8D8CA61BABAB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -140,10 +116,6 @@ Global {0BBEED18-3CC1-4BA6-8E11-603A5D2DEDC9} = {31B4F5EB-E148-4A51-84B5-B90BB8F38AD3} {09135223-6285-4268-BFA3-147BDB5B663B} = {31B4F5EB-E148-4A51-84B5-B90BB8F38AD3} {AB5F75C1-64B7-4E0F-A4B5-B14EB16E6DDC} = {66246A1C-8D45-40FB-A660-C58577122CA7} - {A04C171E-A8E5-40D2-AECA-9FDA6F5FE92B} = {31B4F5EB-E148-4A51-84B5-B90BB8F38AD3} - {183B5AB9-1BD1-4000-97E7-90706D45A668} = {31B4F5EB-E148-4A51-84B5-B90BB8F38AD3} - {81CABDF5-7A9C-44A8-9A0A-2B17DED577A9} = {31B4F5EB-E148-4A51-84B5-B90BB8F38AD3} - {B5B83426-7099-4EAD-9E07-8D8CA61BABAB} = {66246A1C-8D45-40FB-A660-C58577122CA7} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {9F18C25E-E140-43C3-B177-D562E1628370} diff --git a/dockerfile b/dockerfile deleted file mode 100644 index da052071b..000000000 --- a/dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM mcr.microsoft.com/dotnet/sdk:8.0-jammy AS build-env -WORKDIR /app - -RUN apt-get update && apt-get install -y g++ curl cmake - -# Copy everything -COPY . . -# Restore as distinct layers -RUN dotnet restore -# Build and publish a release -RUN dotnet publish ./src/SIL.Machine.Serval.EngineServer/SIL.Machine.Serval.EngineServer.csproj -c Release -o out_engine_server -RUN dotnet publish ./src/SIL.Machine.Serval.JobServer/SIL.Machine.Serval.JobServer.csproj -c Release -o out_job_server - -# Build runtime image -FROM mcr.microsoft.com/dotnet/aspnet:8.0-jammy as production -# libgomp needed for thot -RUN apt-get update && apt-get install -y libgomp1 -WORKDIR /app -COPY --from=build-env /app/out_engine_server ./engine_server -COPY --from=build-env /app/out_job_server ./job_server - -CMD ["bash"] \ No newline at end of file diff --git a/dockerfile.development b/dockerfile.development deleted file mode 100644 index a395da6dd..000000000 --- a/dockerfile.development +++ /dev/null @@ -1,4 +0,0 @@ -FROM mcr.microsoft.com/dotnet/sdk:8.0-jammy -# libgomp needed for thot -RUN apt update && apt install -y unzip libgomp1 && \ - curl -sSL https://aka.ms/getvsdbgsh | /bin/sh /dev/stdin -v latest -l /remote_debugger \ No newline at end of file diff --git a/samples/corpora.ipynb b/samples/corpora.ipynb deleted file mode 100644 index beaa9546d..000000000 --- a/samples/corpora.ipynb +++ /dev/null @@ -1,675 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Text Corpora Tutorial\n", - "\n", - "In this notebook, we will demonstrate how to use Machine to load datasets as text corpora." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
Installed Packages
  • SIL.Scripture, 12.0.1
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "#r \"nuget:SIL.Scripture,12.0.1\"\n", - "#r \"../src/SIL.Machine/bin/Debug/netstandard2.0/SIL.Machine.dll\"\n", - "\n", - "void WriteLine(string text = \"\")\n", - "{\n", - " Console.Write(text + \"\\n\");\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Text Files\n", - "\n", - "Let's start with a simple example of loading a set of text files." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - "using SIL.Machine.Corpora;\n", - "\n", - "var corpus = new TextFileTextCorpus(\"data/en_tok.txt\");" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It is easy to iterate through the sentences in the corpus. We simply iterate on the corpus object." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Would you mind giving us the keys to the room , please ?\n", - "I have made a reservation for a quiet , double room with a telephone and a tv for Rosario Cabedo .\n", - "Would you mind moving me to a quieter room ?\n", - "I have booked a room .\n", - "I think that there is a problem .\n", - "Do you have any rooms with a tv , air conditioning and a safe available ?\n", - "Would you mind showing us a room with a tv ?\n", - "Does it have a telephone ?\n", - "I am leaving on the second at eight in the evening .\n", - "How much does a single room cost per week ?\n" - ] - } - ], - "source": [ - "foreach (var row in corpus.Take(10))\n", - " WriteLine(row.Text);" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Loading Scripture\n", - "\n", - "Machine contains classes for loading Scripture in various formats, such as USFM and USX." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### USX\n", - "\n", - "USX is a common XML format for Scripture. Let's take a look at how to load a set of USX files. First, we create an instance of the `UsxFileTextCorpus` class. We ensure that the correct verse references are used by loading the versification file for this translation. If a versification is not provided, then the English versification is used." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - "using SIL.Scripture;\n", - "\n", - "var versification = Versification.Table.Implementation.Load(\"data/WEB-DBL/release/versification.vrs\", \"web\");\n", - "var corpus = new UsxFileTextCorpus(\"data/WEB-DBL/release/USX_1\", versification);" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's iterate through the corpus. You will notice that each text segment in the corpus has an associated reference. In the case of Scripture, these are `VerseRef` objects." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1JN 1:1: That which was from the beginning, that which we have heard, that which we have seen with our eyes, that which we saw, and our hands touched, concerning the Word of life\n", - "1JN 1:2: (and the life was revealed, and we have seen, and testify, and declare to you the life, the eternal life, which was with the Father, and was revealed to us);\n", - "1JN 1:3: that which we have seen and heard we declare to you, that you also may have fellowship with us. Yes, and our fellowship is with the Father and with his Son, Jesus Christ.\n", - "1JN 1:4: And we write these things to you, that our joy may be fulfilled.\n", - "1JN 1:5: This is the message which we have heard from him and announce to you, that God is light, and in him is no darkness at all.\n", - "1JN 1:6: If we say that we have fellowship with him and walk in the darkness, we lie and don’t tell the truth.\n", - "1JN 1:7: But if we walk in the light as he is in the light, we have fellowship with one another, and the blood of Jesus Christ his Son, cleanses us from all sin.\n", - "1JN 1:8: If we say that we have no sin, we deceive ourselves, and the truth is not in us.\n", - "1JN 1:9: If we confess our sins, he is faithful and righteous to forgive us the sins and to cleanse us from all unrighteousness.\n", - "1JN 1:10: If we say that we haven’t sinned, we make him a liar, and his word is not in us.\n" - ] - } - ], - "source": [ - "foreach (var row in corpus.Take(10))\n", - " WriteLine($\"{row.Ref}: {row.Text}\");" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can also iterate through verses in the corpus by book." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1JN\n", - "======\n", - "1:1: That which was from the beginning, that which we have heard, that which we have seen with our eyes, that which we saw, and our hands touched, concerning the Word of life\n", - "1:2: (and the life was revealed, and we have seen, and testify, and declare to you the life, the eternal life, which was with the Father, and was revealed to us);\n", - "1:3: that which we have seen and heard we declare to you, that you also may have fellowship with us. Yes, and our fellowship is with the Father and with his Son, Jesus Christ.\n", - "\n", - "2JN\n", - "======\n", - "1:1: The elder, to the chosen lady and her children, whom I love in truth, and not I only, but also all those who know the truth,\n", - "1:2: for the truth’s sake, which remains in us, and it will be with us forever:\n", - "1:3: Grace, mercy, and peace will be with us, from God the Father and from the Lord Jesus Christ, the Son of the Father, in truth and love.\n", - "\n", - "3JN\n", - "======\n", - "1:1: The elder to Gaius the beloved, whom I love in truth.\n", - "1:2: Beloved, I pray that you may prosper in all things and be healthy, even as your soul prospers.\n", - "1:3: For I rejoiced greatly when brothers came and testified about your truth, even as you walk in truth.\n", - "\n" - ] - } - ], - "source": [ - "foreach (var text in corpus.Texts)\n", - "{\n", - " WriteLine(text.Id);\n", - " WriteLine(\"======\");\n", - " foreach (var row in text.Take(3))\n", - " {\n", - " var verseRef = (VerseRef)row.Ref;\n", - " WriteLine($\"{verseRef.Chapter}:{verseRef.Verse}: {row.Text}\");\n", - " }\n", - " WriteLine();\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Digital Bible Library Bundles\n", - "\n", - "Now, let's load a Digital Bible Library (DBL) bundle. A DBL bundle is a zip archive that contains all of the data that you need for a publishable Bible translation." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "DBL bundle created.\n" - ] - } - ], - "source": [ - "using System.IO;\n", - "using System.IO.Compression;\n", - "\n", - "Directory.CreateDirectory(\"out\");\n", - "if (File.Exists(\"out/web.zip\"))\n", - " File.Delete(\"out/web.zip\");\n", - "ZipFile.CreateFromDirectory(\"data/WEB-DBL\", \"out/web.zip\");\n", - "WriteLine(\"DBL bundle created.\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we create a `DblBundleTextCorpus` instance. There is no need to specify versification, because the `DblBundleTextCorpus` class takes care of that for us." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - "var corpus = new DblBundleTextCorpus(\"out/web.zip\");" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can iterate through the corpus just as we did before. All text corpus classes in Machine adhere to the same interface, so it is easy to switch between the various classes. Also, you can see that the verse text is nicely tokenized." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1JN 1:1: That which was from the beginning, that which we have heard, that which we have seen with our eyes, that which we saw, and our hands touched, concerning the Word of life\n", - "1JN 1:2: (and the life was revealed, and we have seen, and testify, and declare to you the life, the eternal life, which was with the Father, and was revealed to us);\n", - "1JN 1:3: that which we have seen and heard we declare to you, that you also may have fellowship with us. Yes, and our fellowship is with the Father and with his Son, Jesus Christ.\n", - "1JN 1:4: And we write these things to you, that our joy may be fulfilled.\n", - "1JN 1:5: This is the message which we have heard from him and announce to you, that God is light, and in him is no darkness at all.\n", - "1JN 1:6: If we say that we have fellowship with him and walk in the darkness, we lie and don’t tell the truth.\n", - "1JN 1:7: But if we walk in the light as he is in the light, we have fellowship with one another, and the blood of Jesus Christ his Son, cleanses us from all sin.\n", - "1JN 1:8: If we say that we have no sin, we deceive ourselves, and the truth is not in us.\n", - "1JN 1:9: If we confess our sins, he is faithful and righteous to forgive us the sins and to cleanse us from all unrighteousness.\n", - "1JN 1:10: If we say that we haven’t sinned, we make him a liar, and his word is not in us.\n" - ] - } - ], - "source": [ - "foreach (var row in corpus.Take(10))\n", - " WriteLine($\"{row.Ref}: {row.Text}\");" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Paratext Projects\n", - "\n", - "Another useful text corpus class is `ParatextTextCorpus`. This class is used to load a Paratext project. It properly loads the configured encoding and versification for the project." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - "var corpus = new ParatextTextCorpus(\"data/WEB-PT\");" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now, let's iterate through the segments." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1JN 1:1: That which was from the beginning, that which we have heard, that which we have seen with our eyes, that which we saw, and our hands touched, concerning the Word of life\n", - "1JN 1:2: (and the life was revealed, and we have seen, and testify, and declare to you the life, the eternal life, which was with the Father, and was revealed to us);\n", - "1JN 1:3: that which we have seen and heard we declare to you, that you also may have fellowship with us. Yes, and our fellowship is with the Father and with his Son, Jesus Christ.\n", - "1JN 1:4: And we write these things to you, that our joy may be fulfilled.\n", - "1JN 1:5: This is the message which we have heard from him and announce to you, that God is light, and in him is no darkness at all.\n", - "1JN 1:6: If we say that we have fellowship with him and walk in the darkness, we lie and don’t tell the truth.\n", - "1JN 1:7: But if we walk in the light as he is in the light, we have fellowship with one another, and the blood of Jesus Christ his Son, cleanses us from all sin.\n", - "1JN 1:8: If we say that we have no sin, we deceive ourselves, and the truth is not in us.\n", - "1JN 1:9: If we confess our sins, he is faithful and righteous to forgive us the sins and to cleanse us from all unrighteousness.\n", - "1JN 1:10: If we say that we haven’t sinned, we make him a liar, and his word is not in us.\n" - ] - } - ], - "source": [ - "foreach (var row in corpus.Take(10))\n", - " WriteLine($\"{row.Ref}: {row.Text}\");" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Parallel Text Corpora\n", - "\n", - "So far we have only dealt with monolingual corpora. For many tasks, such as machine translation, parallel corpora are required. Machine provides a corpus class for combining two monolingual corpora into a parallel corpus." - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In order to create a parallel text corpus, we must first create the source and target monolingual text corpora. Then, we create the parallel corpus using the `AlignRows` method." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - "var sourceCorpus = new ParatextTextCorpus(\"data/VBL-PT\");\n", - "var targetCorpus = new ParatextTextCorpus(\"data/WEB-PT\");\n", - "var parallelCorpus = sourceCorpus.AlignRows(targetCorpus);" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can now iterate through the parallel segments." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1JN 1:1\n", - "Source: Esta carta trata sobre la Palabra de vida que existía desde el principio, que hemos escuchado, que hemos visto con nuestros propios ojos y le hemos contemplado, y que hemos tocado con nuestras manos.\n", - "Target: That which was from the beginning, that which we have heard, that which we have seen with our eyes, that which we saw, and our hands touched, concerning the Word of life\n", - "1JN 1:2\n", - "Source: Esta Vida nos fue revelada. La vimos y damos testimonio de ella. Estamos hablándoles de Aquél que es la Vida Eterna, que estaba con el Padre, y que nos fue revelado.\n", - "Target: (and the life was revealed, and we have seen, and testify, and declare to you the life, the eternal life, which was with the Father, and was revealed to us);\n", - "1JN 1:3\n", - "Source: Los que hemos visto y oído eso mismo les contamos, para que también puedan participar de esta amistad junto a nosotros. Esta amistad con el Padre y su Hijo Jesucristo.\n", - "Target: that which we have seen and heard we declare to you, that you also may have fellowship with us. Yes, and our fellowship is with the Father and with his Son, Jesus Christ.\n", - "1JN 1:4\n", - "Source: Escribimos para decirles esto, a fin de que nuestra felicidad sea completa.\n", - "Target: And we write these things to you, that our joy may be fulfilled.\n", - "1JN 1:5\n", - "Source: Este es el mensaje que recibimos de él y que nosotros les declaramos a ustedes: Dios es luz, y no hay ningún vestigio de oscuridad en él.\n", - "Target: This is the message which we have heard from him and announce to you, that God is light, and in him is no darkness at all.\n" - ] - } - ], - "source": [ - "foreach (var row in parallelCorpus.Take(5))\n", - "{\n", - " WriteLine($\"{row.Ref}\");\n", - " WriteLine($\"Source: {row.SourceText}\");\n", - " WriteLine($\"Target: {row.TargetText}\");\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Corpus Processing\n", - "\n", - "Often a text corpus must be processed in some way as a part of a AI/ML pipeline. Machine has a set of operations that can be used to process a corpus easily. Lowercasing text is a common pre-processing step, so let's show how to apply the \"lowercase\" operation." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1JN 1:1: that which was from the beginning, that which we have heard, that which we have seen with our eyes, that which we saw, and our hands touched, concerning the word of life\n", - "1JN 1:2: (and the life was revealed, and we have seen, and testify, and declare to you the life, the eternal life, which was with the father, and was revealed to us);\n", - "1JN 1:3: that which we have seen and heard we declare to you, that you also may have fellowship with us. yes, and our fellowship is with the father and with his son, jesus christ.\n", - "1JN 1:4: and we write these things to you, that our joy may be fulfilled.\n", - "1JN 1:5: this is the message which we have heard from him and announce to you, that god is light, and in him is no darkness at all.\n", - "1JN 1:6: if we say that we have fellowship with him and walk in the darkness, we lie and don’t tell the truth.\n", - "1JN 1:7: but if we walk in the light as he is in the light, we have fellowship with one another, and the blood of jesus christ his son, cleanses us from all sin.\n", - "1JN 1:8: if we say that we have no sin, we deceive ourselves, and the truth is not in us.\n", - "1JN 1:9: if we confess our sins, he is faithful and righteous to forgive us the sins and to cleanse us from all unrighteousness.\n", - "1JN 1:10: if we say that we haven’t sinned, we make him a liar, and his word is not in us.\n" - ] - } - ], - "source": [ - "var corpus = new ParatextTextCorpus(\"data/WEB-PT\");\n", - "\n", - "foreach (var row in corpus.Lowercase().Take(10))\n", - " WriteLine($\"{row.Ref}: {row.Text}\");" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Multiple operations can be chained together. Here we will tokenize, lowercase, and normalize the corpus." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1JN 1:1: that which was from the beginning , that which we have heard , that which we have seen with our eyes , that which we saw , and our hands touched , concerning the word of life\n", - "1JN 1:2: ( and the life was revealed , and we have seen , and testify , and declare to you the life , the eternal life , which was with the father , and was revealed to us ) ;\n", - "1JN 1:3: that which we have seen and heard we declare to you , that you also may have fellowship with us . yes , and our fellowship is with the father and with his son , jesus christ .\n", - "1JN 1:4: and we write these things to you , that our joy may be fulfilled .\n", - "1JN 1:5: this is the message which we have heard from him and announce to you , that god is light , and in him is no darkness at all .\n", - "1JN 1:6: if we say that we have fellowship with him and walk in the darkness , we lie and don’t tell the truth .\n", - "1JN 1:7: but if we walk in the light as he is in the light , we have fellowship with one another , and the blood of jesus christ his son , cleanses us from all sin .\n", - "1JN 1:8: if we say that we have no sin , we deceive ourselves , and the truth is not in us .\n", - "1JN 1:9: if we confess our sins , he is faithful and righteous to forgive us the sins and to cleanse us from all unrighteousness .\n", - "1JN 1:10: if we say that we haven’t sinned , we make him a liar , and his word is not in us .\n" - ] - } - ], - "source": [ - "using SIL.Machine.Tokenization;\n", - "\n", - "foreach (var row in corpus.Tokenize().Lowercase().NfcNormalize().Take(10))\n", - " WriteLine($\"{row.Ref}: {row.Text}\");" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Corpus processing operations are also available on parallel corpora." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1JN 1:1\n", - "Source: esta carta trata sobre la palabra de vida que existía desde el principio , que hemos escuchado , que hemos visto con nuestros propios ojos y le hemos contemplado , y que hemos tocado con nuestras manos .\n", - "Target: that which was from the beginning , that which we have heard , that which we have seen with our eyes , that which we saw , and our hands touched , concerning the word of life\n", - "1JN 1:2\n", - "Source: esta vida nos fue revelada . la vimos y damos testimonio de ella . estamos hablándoles de aquél que es la vida eterna , que estaba con el padre , y que nos fue revelado .\n", - "Target: ( and the life was revealed , and we have seen , and testify , and declare to you the life , the eternal life , which was with the father , and was revealed to us ) ;\n", - "1JN 1:3\n", - "Source: los que hemos visto y oído eso mismo les contamos , para que también puedan participar de esta amistad junto a nosotros . esta amistad con el padre y su hijo jesucristo .\n", - "Target: that which we have seen and heard we declare to you , that you also may have fellowship with us . yes , and our fellowship is with the father and with his son , jesus christ .\n", - "1JN 1:4\n", - "Source: escribimos para decirles esto , a fin de que nuestra felicidad sea completa .\n", - "Target: and we write these things to you , that our joy may be fulfilled .\n", - "1JN 1:5\n", - "Source: este es el mensaje que recibimos de él y que nosotros les declaramos a ustedes : dios es luz , y no hay ningún vestigio de oscuridad en él .\n", - "Target: this is the message which we have heard from him and announce to you , that god is light , and in him is no darkness at all .\n" - ] - } - ], - "source": [ - "var sourceCorpus = new ParatextTextCorpus(\"data/VBL-PT\");\n", - "var targetCorpus = new ParatextTextCorpus(\"data/WEB-PT\");\n", - "var parallelCorpus = sourceCorpus.AlignRows(targetCorpus);\n", - "\n", - "foreach (var row in parallelCorpus.Tokenize().Lowercase().NfcNormalize().Take(5))\n", - "{\n", - " WriteLine($\"{row.Ref}\");\n", - " WriteLine($\"Source: {row.SourceText}\");\n", - " WriteLine($\"Target: {row.TargetText}\");\n", - "}" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".NET (C#)", - "language": "C#", - "name": ".net-csharp" - }, - "language_info": { - "name": "C#" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/samples/data/VBL-PT/631JNspavbl.SFM b/samples/data/VBL-PT/631JNspavbl.SFM deleted file mode 100644 index 98fa9e3b1..000000000 --- a/samples/data/VBL-PT/631JNspavbl.SFM +++ /dev/null @@ -1,138 +0,0 @@ -\id 1JN spa Versión Biblia Libre -\h 1 Juan -\toc1 1 Juan -\toc2 1 Juan -\mt1 1 Juan -\c 1 -\p -\v 1 Esta carta trata sobre la Palabra de vida que existía desde el principio, que hemos escuchado, que hemos visto con nuestros propios ojos y le hemos contemplado, y que hemos tocado con nuestras manos\f + \fr 1.1 \ft La estructura griega de la oración se ha ajustado para darle sentido. \f*. -\v 2 Esta Vida nos fue revelada. La vimos y damos testimonio de ella. Estamos hablándoles de Aquél que es la Vida Eterna, que estaba con el Padre, y que nos fue revelado. -\v 3 Los que hemos visto y oído eso mismo les contamos, para que también puedan participar de esta amistad\f + \fr 1.3 \ft Literalmente, “compañerismo.”\f* junto a nosotros. Esta amistad con el Padre y su Hijo Jesucristo. -\v 4 Escribimos para decirles esto, a fin de que nuestra felicidad sea completa. -\p -\v 5 Este es el mensaje que recibimos de él y que nosotros les declaramos a ustedes: Dios es luz, y no hay ningún vestigio de oscuridad en él\f + \fr 1.5 \ft En griego hay una doble negación para hacer énfasis, literalmente, “la oscuridad en él no existe, de ninguna manera.”\f*. -\v 6 Si decimos ser sus amigos, y seguimos viviendo\f + \fr 1.6 \ft Literalmente, “caminando.” Ver también 1:7.\f* en oscuridad, estamos mintiendo, y no vivimos en la verdad. -\v 7 Pero si vivimos en la luz, así como él está en la luz, entonces somos amigos unos con otros, y la sangre de Jesús, su Hijo, nos limpia de todo pecado. -\p -\v 8 Si decimos que no pecamos, nos engañamos a nosotros mismos, y la verdad no está en nosotros. -\v 9 Pero si confesamos nuestros pecados, él es fiel y justo para perdonar nuestros pecados y limpiarnos de todo lo malo que hay dentro de nosotros. -\v 10 Si decimos que no hemos pecado, estamos llamando a Dios mentiroso, y su palabra no está en nosotros. -\c 2 -\p -\v 1 Queridos hijos míos, les escribo esto para que no pequen. Pero si alguno peca, tenemos a alguien que nos defiende ante el Padre, a Jesucristo, que es verdaderamente justo. -\v 2 Por él son perdonados nuestros pecados, y no solo los nuestros, sino los de todo el mundo. -\v 3 Podemos estar seguros de que lo conocemos si seguimos sus mandamientos. -\v 4 Todo el que dice: “Yo conozco a Dios,” pero no hace su voluntad, es mentiroso, y no tiene la verdad. -\v 5 Pero los que siguen la palabra de Dios permiten que su amor llene sus corazones por completo. Así es como sabemos que vivimos en él. -\v 6 Todo el que dice vivir en él, debe vivir como Jesús vivió. -\p -\v 7 Amigos, no les escribo para darles un nuevo mandamiento, sino un mandamiento antiguo que ya teníamos desde el principio. Este mandamiento antiguo ya lo han escuchado. -\v 8 Pero en cierto sentido les estoy dando un nuevo mandamiento. Su verdad se revela en Jesús y en ustedes, pues viene el fin de la oscuridad y la luz verdadera ya está brillando. -\v 9 Los que dicen que viven en la luz pero aborrecen a un hermano cristiano\f + \fr 2.9 \ft Literalmente, “hermano.”\f* todavía tienen tinieblas dentro de sí. -\v 10 Los que aman a sus hermanos cristianos viven en la luz, y no hacen pecar a otros\f + \fr 2.10 \ft Literalmente, “no hay engaño.” En otras palabras, algo que hace tropezar a otros.\f*. -\v 11 Los que aborrecen a un hermano cristiano están en oscuridad. Tropiezan en la oscuridad, sin saber hacia dónde van porque la oscuridad los ha cegado. -\p -\v 12 Queridos amigos, les escribo a ustedes, hijos\f + \fr 2.12 \ft Juan identifica tres grupos: hijos, padres y jóvenes. Probablemente se refiere a las distintas edades de la vida cristiana, más que a grupos literales. \f*, porque sus pecados han sido perdonados por el nombre de Jesús. -\v 13 Les escribo a ustedes, padres, porque ustedes lo conocen a él, que ha existido desde el principio. Les escribo a ustedes, jóvenes, porque han vencido el mal. -\v 14 Les escribo a ustedes, pequeñitos, porque ustedes conocen al Padre. Les escribo a ustedes, Padres, porque conocen al que ha existido desde el principio. Les escribo a ustedes, jóvenes, porque son fuertes. Porque la palabra de Dios vive en ustedes, y han vencido al maligno. -\p -\v 15 No amen al mundo, ni anhelen las cosas que hay en él. Si aman al mundo, no tendrán el amor del Padre en ustedes. -\v 16 Porque todas las cosas de este mundo, nuestros deseos pecaminosos, nuestro deseo por todo lo que vemos, nuestra jactancia por lo que hemos logrado en la vida, ninguna de esas cosas viene del Padre, sino del mundo. -\v 17 El mundo y sus malos deseos acabarán, pero los que hacen la voluntad de Dios vivirán para siempre. -\p -\v 18 Queridos amigos, esta es la última hora. Como han escuchado, el anticristo viene. Y ya han venido muchos anticristos. Así es como sabemos que esta es la última hora. -\v 19 Ellos se fueron, pero no eran parte de nosotros, porque si así hubiera sido, habrían permanecido aquí. Pero cuando se fueron demostraron que ninguno de ellos hacía parte de nosotros. -\p -\v 20 Pero ustedes han sido ungidos\f + \fr 2.20 \ft Ungir es el acto de derramar un líquido (a menudo aceite) sobre la cabeza de alguien para indicar que esa persona tiene una bendición especial y está apartada para un rol particular (como el reinado en el Antiguo Testamento). Aquí el ungimiento se refiere a la bendición del Espíritu Santo, quien en palabras de Jesús, nos conduce a toda verdad.\f* con la bendición del Espíritu Santo, y todos ustedes saben lo que es verdad. -\v 21 No les escribo porque no conozcan la verdad, sino precisamente porque la conocen, y porque no hay engaño en ella. -\v 22 ¿Quién es el mentiroso? Todo aquél que niega que Jesús es el Cristo\f + \fr 2.22 \ft Quiere decir Mesías. (Cristo, en griego). Ambos se refieren al que es ungido.\f*. El anticristo es todo aquél que niega al Padre y al Hijo. -\v 23 Todo aquél que niega al Hijo, tampoco tiene al Padre; y todo el que reconoce al Hijo, tiene al Padre también. -\v 24 En cuanto a ustedes, asegúrense de que lo que oyeron desde el principio siga vivo en ustedes. Si lo que oyeron desde el principio vive en ustedes, también vivirán en el Hijo y en el Padre. -\v 25 La vida eterna. ¡Eso es lo que nos ha prometido! -\p -\v 26 Escribo esto para advertirles contra las cosas que quieren descarriarlos. -\v 27 Pero el ungimiento que recibieron de él por medio del Espíritu\f + \fr 2.27 \ft El espíritu, según el versículo 20. \f* vive en ustedes, y no necesitan que nadie los enseñe. El ungimiento del Espíritu les enseña todas las cosas. Esa es la verdad. No es una mentira. Así que vivan en Cristo, como se les ha enseñado. -\p -\v 28 Ahora, mis queridos amigos, sigan viviendo en Cristo, para que cuando aparezca, podamos estar seguros y no tengamos vergüenza delante él en su venida. -\v 29 Si ustedes saben que él es bueno y justo\f + \fr 2.29 \ft Literalmente, “justo.” Sin embargo, esta palabra a menudo solo se usa en un sentido religioso hoy y no tiene mucho significado en el hablar cotidiano.\f*, entonces también deben saber que todo el que hace lo justo ha nacido de Dios. -\c 3 -\p -\v 1 ¡Miren el amor que tiene el Padre para con nosotros! Por eso podemos ser llamados hijos de Dios, ¡porque eso es lo que somos! La razón por la que el mundo no nos reconoce como hijos de Dios es porque no lo reconocen a él. -\v 2 Amigos míos, ya somos hijos de Dios, pero lo que llegaremos a ser no se ha revelado todavía. Pero sabemos que cuando él aparezca seremos como él, porque lo veremos como él es realmente. -\v 3 Todos los que tienen esta esperanza en él, asegúrense de ser puros, como él lo es. -\v 4 Todos los que pecan son violadores de la ley de Dios. -\v 5 Pero desde luego ustedes saben que Jesús vino para eliminar los pecados, y en él no hay pecado. -\v 6 Todos los que viven en él, ya no pecan más; todos los que siguen pecando es porque no lo han visto y no lo han conocido. -\p -\v 7 Queridos amigos, no dejen que nadie los engañe: los que hacen justicia son justos, así como Jesús es justo. -\v 8 Los que pecan son del diablo, porque el diablo ha estado pecando desde el principio. Por eso vino el Hijo de Dios, para destruir lo que el diablo ha hecho. -\v 9 Y todos los que son nacidos de Dios ya no pecan más, porque la naturaleza de Dios\f + \fr 3.9 \ft Literalmente, “su semilla.”\f* habita en ellos. Y no pueden seguir pecando porque han nacido de Dios. -\v 10 Así es como podemos distinguir a los hijos de Dios y los hijos del diablo: todos aquellos que no obran con justicia, no pertenecen a Dios, ni aquellos que no aman a sus hermanos cristianos. -\p -\v 11 El mensaje que han escuchado desde el principio es este: debemos amarnos unos a otros. -\v 12 No podemos ser como Caín, que pertenecía al maligno, y mató a su hermano. ¿Por qué lo mató? Porque Caín era malo, pero su hermano era justo. -\v 13 Así que no se sorprendan si este mundo los aborrece. -\p -\v 14 La razón por la que sabemos que hemos ido de la muerte a la vida es porque amamos a nuestros hermanos y hermanas en la fe. Porque el que no ama sigue muerto. -\v 15 Los que odian a sus hermanos cristianos son asesinos, y ustedes saben que los asesinos no tendrán vida eterna con ellos. -\v 16 Así es como sabemos qué es el amor: Jesús entregó su vida por nosotros, y nosotros debemos entregar nuestras vidas por nuestros hermanos en la fe. -\v 17 Si alguno de ustedes vive cómodamente en este mundo, y ve a su hermano o hermana en Cristo padeciendo necesidad, pero no tiene compasión, ¿cómo podemos decir que el amor vive en ustedes? -\v 18 Queridos amigos, no digamos que amamos solo con palabras, sino mostremos nuestro amor en lo que hacemos y en la manera como demostramos la verdad. -\v 19 Así es como sabremos que pertenecemos a la verdad, y pondremos nuestras mentes\f + \fr 3.19 \ft Literalmente, “corazones.” Se creía que el corazón era el órgano con el que se pensaba.\f* en paz con Dios -\v 20 cuando pensemos que estamos en error. Dios es más grande de lo que creemos, y lo sabe todo. -\v 21 Así que, queridos amigos, si tenemos la tranquilidad de que no estamos en el error, podemos tener confianza ante Dios. -\v 22 Pues recibiremos de él cualquier cosa que le pidamos, porque seguimos sus mandamientos y hacemos lo que le agrada. -\v 23 Y esto es lo que él manda: que debemos confiar en el nombre\f + \fr 3.23 \ft Nombre, en el sentido del carácter y la reputación, más que un nombre asignado.\f* de su Hijo Jesucristo, y amarnos unos a otros, así como él nos mandó. -\v 24 Los que guardan sus mandamientos siguen viviendo en él, y él vive en ellos. Y sabemos que él vive en nosotros por el Espíritu que nos ha dado. -\c 4 -\p -\v 1 Queridos amigos, no confien en todos los espíritus, sino pruébenlos para saber si son o no de Dios, porque hay muchos falsos profetas en este mundo. -\v 2 ¿Cómo pueden reconocer el Espíritu de Dios? Pues todo espíritu que acepta que Jesús vino en carne humana, es de Dios; -\v 3 pero todo espíritu que no acepta a Jesús, ese espíritu no es de Dios. De hecho, es el espíritu del anticristo, del cual oyeron que vendrá, y que ya está en el mundo. -\p -\v 4 Pero ustedes pertenecen a Dios, mis amigos, y los han vencido\f + \fr 4.4 \ft Refiriéndose de nuevo a los falsos profetas y al espíritu que los inspira.\f*, porque el que está en ustedes es más grande que el que está en el mundo. -\v 5 Ellos pertenecen al mundo, y hablan como personas del mundo, y el mundo los oye. -\v 6 Sin embargo, nosotros pertenecemos a Dios y todo el que conoce a Dios, nos escucha; pero los que no pertenecen a Dios, no nos escuchan. Así es como podemos distinguir el espíritu de verdad del espíritu de engaño. -\p -\v 7 Queridos amigos, sigamos amándonos unos a otros, porque el amor viene de Dios. Todos los que aman son nacidos de Dios y conocen a Dios. -\v 8 Los que no aman, no conocen a Dios, porque Dios es amor. -\v 9 ¿Cómo nos fue demostrado el amor de Dios? Dios envió a su único Hijo para que viviéramos por él. -\v 10 ¡Eso es amor! No es que nosotros hayamos amado a Dios, sino que él nos amó y envió a su Hijo para ser la reconciliación por nuestros pecados. -\p -\v 11 Amigos, si esta es la manera como Dios nos ama, debemos amarnos unos a otros de esta misma manera. -\v 12 Nadie ha visto a Dios. Sin embargo, si nos amamos unos a otros, entonces Dios vive en nosotros, y su amor se cumple en nosotros. -\v 13 ¿Cómo podemos saber que él vive en nosotros? En que nos ha dado el poder de amar\f + \fr 4.13 \ft Implícito.\f* por su Espíritu. -\p -\v 14 Porque somos testigos de lo que hemos visto y testificamos que el Padre envió al Hijo como Salvador del mundo. -\v 15 Dios vive en todos los que declaran que Jesús es el Hijo de Dios, y ellos viven en Dios. -\v 16 Hemos experimentado y creído en el amor que Dios tiene por nosotros. Dios es amor, y los que viven en amor, viven en Dios, y Dios en ellos. -\v 17 Es así como el amor se completa en nosotros, para que podamos estar seguros en el día del juicio: por el hecho de que vivimos como él en este mundo. -\v 18 Donde hay amor no puede haber temor. Y Dios nos ama por completo, y este amor echa fuera todos nuestros miedos. Si tememos, es porque tememos ser castigados, y eso muestra que no hemos sido plenamente transformados por la plenitud del amor de Dios. -\v 19 Nosotros amamos porque él nos amó primero. -\v 20 Los que dicen: “Yo amo a Dios,” pero odian a su hermano o hermana en la fe, son mentirosos. Los que no aman a un hermano al que pueden ver, no pueden amar a Dios, a quien no ven. -\v 21 Este es el mandamiento que nos dio: los que aman a Dios, amen también a sus hermanos. -\c 5 -\p -\v 1 Todo el que cree que Jesús es el Cristo nacido de Dios, y el que ama al Padre también ama a su hijo. -\v 2 ¿Cómo sabemos que amamos a los hijos de Dios? Cuando amamos a Dios y seguimos sus mandamientos. -\v 3 Amar a Dios quiere decir que seguimos sus mandamientos, y sus mandamientos no son una carga pesada. -\v 4 Todo el que nace de Dios vence al mundo. La manera como obtenemos la victoria y vencemos al mundo es por la fe en Dios. -\v 5 ¿Quién puede vencer al mundo? Solo los que creen en Jesús, creyendo que él es el Hijo de Dios. -\v 6 Él es el que vino por agua y sangre, Jesucristo. No solo vino por agua, sino por agua y sangre\f + \fr 5.6 \ft Esto a menudo se interpreta con el fin de dar el significado del agua del bautismo y la sangre que significa su muerte.\f*. El Espíritu prueba y confirma esto, porque el Espíritu es la verdad. -\v 7 Asó que hay tres que dan evidencia de ello: -\v 8 el Espíritu, el agua, y la sangre, y los tres están de acuerdo como si fueran uno\f + \fr 5.8 \ft 5:7, 8. Se debate sobre la autenticidad de los versículos 7 y 8.\f*. -\p -\v 9 Si aceptamos la evidencia que dan los testigos humanos, entonces la evidencia que da Dios es más importante. La evidencia que Dios da es su testimonio sobre su Hijo. -\v 10 Los que creen en el Hijo de Dios han aceptado y se han aferrado a esta evidencia. Los que no creen en Dios, llaman a Dios mentirosos, porque no creen la evidencia que Dios da sobre su Hijo. -\v 11 Y la evidencia es esta: Dios nos ha dado vida eterna por medio de su Hijo. -\v 12 Todo el que tiene al Hijo tiene vida; y quien no tiene al Hijo no tiene vida. -\v 13 Escribo para decirles a los que entre ustedes creen en el nombre del Hijo de Dios, para que puedan estar seguros que tienen la vida eterna. -\v 14 Podemos estar seguros de que él nos escuchará siempre y cuando pidamos conforme a su voluntad. -\v 15 Si sabemos que él oye nuestras peticiones, podemos estar seguros de que recibiremos lo que le pedimos. -\v 16 Si ves a tu hermano en la fe cometiendo un pecado que no es mortal\f + \fr 5.16 \ft Pecado mortal, literalmente “un pecado para muerte.”\f*, debes orar y Dios le otorgará vida al que ha pecado. (Pero no por un pecado mortal. Porque hay un pecado que es mortal, y no quiero decir que la gente deba orar por eso. -\v 17 Sí, todo lo que no es recto es pecado, pero hay un pecado que no es mortal). -\p -\v 18 Reconocemos que los que nacen de Dios no siguen pecando más. El Hijo de Dios\f + \fr 5.18 \ft Literalmente, El Único que es nacido de Dios, siguiendo el concepto que está al principio del versículo. En el siguiente versículo queda clara su identificación.\f* los protege y el diablo no puede hacerles daño. -\v 19 Pues sabemos que pertenecemos a Dios, y que el mundo está bajo control del maligno. -\v 20 También sabemos que el Hijo de Dios ha venido, y nos ha ayudado a entender, para que podamos reconocer al que es verdadero. Vivimos en él, que es verdadero, en su Hijo Jesucristo. Él es el verdadero Dios, y es vida eterna\f + \fr 5.20 \ft Puede entenderse de manera que él da la vida eterna pero también que él vive eternamente.\f*. -\v 21 Amigos queridos, aléjense del culto a los ídolos. diff --git a/samples/data/VBL-PT/642JNspavbl.SFM b/samples/data/VBL-PT/642JNspavbl.SFM deleted file mode 100644 index f6177e5ee..000000000 --- a/samples/data/VBL-PT/642JNspavbl.SFM +++ /dev/null @@ -1,24 +0,0 @@ -\id 2JN spa Versión Biblia Libre -\h 2 Juan -\toc1 2 Juan -\toc2 2 Juan -\mt1 2 Juan -\c 1 -\p -\v 1 Esta carta viene de parte del anciano\f + \fr 1.1 \ft 1. Se entiende que es Juan.\f* a la dama escogida\f + \fr 1.1 \ft 1. La “dama escogida” podría referirse a la iglesia, o a una mujer Cristiana en particular. Lo mismo ocurre en el versículo 13.\f* y a sus hijos, a quienes amo en la verdad. No solo de mi parte, sino de parte de todos los que conocen la verdad, -\v 2 porque la verdad vive en nosotros y estará con nosotros para siempre. -\v 3 Que la gracia, la misericordia y la paz sigan con nosotros, de parte de Dios el Padre y de Jesucristo, el Hijo del Padre, en verdad y en amor. -\p -\v 4 Me alegré al saber que algunos de tus hijos están siguiendo la verdad, como el Padre nos mandó. -\v 5 Ahora te digo, apreciada hermana, no como una instrucción, sino siguiendo lo que entendimos desde el principio, que debemos amarnos unos a otros. -\v 6 Y esto es el amor: que sigamos los mandamientos de Dios. Y el mandamiento, como oíste desde el principio, es que debemos vivir en amor\f + \fr 1.6 \ft 6. O, “que debemos comportarnos de esta manera.”\f*. -\p -\v 7 Te digo estas cosas porque han salido muchos engañadores al mundo. Personas que no aceptan que Jesucristo ha venido como ser humano. Y cualquiera que sea así es un engañador y anticristo. -\v 8 Ten cuidado de no perder aquello por lo que hemos trabajado tanto, y que recibas lo que te corresponde. -\p -\v 9 Todos aquellos que se vuelven extremistas\f + \fr 1.9 \ft 9. Literalmente, “que van demasiado lejos.”\f* y no siguen la enseñanza de Cristo, ni tienen a Dios. Los que siguen la enseñanza de Cristo tienen tanto al Padre como al Hijo en ellos. -\v 10 Si la gente se acerca a ti y no dan evidencias de la enseñanza de Cristo, no los recibas no los aceptes, -\v 11 porque si los animas, participas de sus malas obras. -\p -\v 12 Tengo tantas cosas que decirte pero que no las escribiré más con tinta, porque espero visitarte y hablar contigo cara a cara. ¡Eso nos haría muy felices! -\v 13 Saludos de los hijos de tu hermana escogida. diff --git a/samples/data/VBL-PT/653JNspavbl.SFM b/samples/data/VBL-PT/653JNspavbl.SFM deleted file mode 100644 index 5f723c138..000000000 --- a/samples/data/VBL-PT/653JNspavbl.SFM +++ /dev/null @@ -1,25 +0,0 @@ -\id 3JN spa Versión Biblia Libre -\h 3 Juan -\toc1 3 Juan -\toc2 3 Juan -\mt1 3 Juan -\c 1 -\p -\v 1 Esta carta viene de parte del anciano\f + \fr 1.1 \ft 1. A menudo esto se refiere a Juan, el escritor de esta carta.\f* a Gayo, mi querido amigo, a quien amo en la verdad. -\v 2 Amigo mío, por encima de todas las cosas oro para que estés bien y tengas buena salud, pues sé que espiritualmente estás bien. -\p -\v 3 Me alegré cuando vinieron algunos hermanos creyentes y me hablaron sobre tu compromiso con la verdad, y sobre cómo sigues viviendo conforme a ella. -\v 4 Nada me alegra más que escuchar que mis amados hermanos siguen la verdad. -\v 5 Querido amigo, tu fidelidad se demuestra en lo que haces al cuidar de los hermanos\f + \fr 1.5 \ft 5. Los creyentes cristianos que viajaban por el mundo predicando la buena noticia.\f*, incluso los que no conoces. -\v 6 Ellos han dado buen testimonio de tu amor ante la iglesia. Por favor, al enviarlos de camino, trátalos de un modo que agrade a Dios, -\v 7 pues viajan en su nombre, y no aceptan nada de los que no son creyentes\f + \fr 1.7 \ft 7. Literalmente, “gentiles.”\f*. -\v 8 Debemos apoyarlos para juntos ser partícipes de la verdad. -\p -\v 9 Le escribí sobre esto a la iglesia, pero Diótrefes, a quien le gusta hacerse cargo de las cosas\f + \fr 1.9 \ft 9. Literalmente, “le encanta ser el primero.”\f*, se niega a aceptar nuestra autoridad. -\v 10 Así que si logro ir a visitarlos, dejaré claro lo que él ha estado haciendo. Pues ha estado acusándonos falsamente. Y no satisfecho con eso, se niega a recibir a algún otro hermano. Tampoco permitirá que otros los reciban, y expulsa de la iglesia a quienes lo hacen. -\p -\v 11 Amigo mío, no imites el mal, sino el bien. Los que hacen el bien le pertenecen a Dios; los que hacen el mal no lo conocen\f + \fr 1.11 \ft 11. Literalmente, “no han visto a Dios.”\f*. -\p -\v 12 Todos hablan bien de Demetrio, ¡y la verdad también habla bien de él! Nosotros también hablamos bien de él, y ustedes saben que decimos la verdad. -\v 13 Tengo mucho que decirte, pero no quiero hacerlo escribiendo con tinta. -\v 14 Espero verte pronto para que hablemos cara a cara. ¡Que la paz esté contigo! Los amigos que están aquí te envían sus saludos. Te ruego que saludes personalmente, y por nombre, a nuestros amigos de allá. diff --git a/samples/data/VBL-PT/BookNames.xml b/samples/data/VBL-PT/BookNames.xml deleted file mode 100644 index 194238f0e..000000000 --- a/samples/data/VBL-PT/BookNames.xml +++ /dev/null @@ -1,126 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/samples/data/VBL-PT/Settings.xml b/samples/data/VBL-PT/Settings.xml deleted file mode 100644 index 67a70657c..000000000 --- a/samples/data/VBL-PT/Settings.xml +++ /dev/null @@ -1,71 +0,0 @@ - - usfm.sty - 000000000000000000000000000000000000000111111111111111111111111111000000000000000000000000000000000100000000000000000000000 - 4 - es:Latn:: - Spanish - 8.0.63.1 - Versión Biblia Libre [spa] Spanish Free Bible Version - 65001 - T - <p>copyright © 2018 <a href="http://www.freebibleversion.org/">Jonathan Gallagher y Shelly Barrios de Avila</a>. Esta obra se encuentra bajo una licencia de Creative Commons Attribution-NoDerivs Unported License. La traducción Free Bible Version está siendo distribuida para uso libre, con las únicas condiciones siguientes: a) debe ser identificada como Free BibleVersion, y b) el texto no debe ser cambiado.</p> - spavbl - 482ddd53705278cce28a22c6292957b4d0c095c1 - Gentium - 14 - - - fbx9s8quxQ4Ec38m4 - Public - - spavbl.SFM - 41MAT - - NFC - Major::BiblicalTerms.xml - F - F - F - Standard:: - - /home/kahunapule/sync/BibleConv/output/spavbl/usfm - . - . - ShortName - ShortName - NoteInAnyVerse - T - False - False - False - - - , - — - ; - ; - - True - - - ‌ ‍ õ [ ] @ Á Ú / k w – … ü - Z — ( ) V 9 G R 8 B 7 0 O F I É 6 ; U 5 4 Q ’ ‘ 3 H 2 x ¡ ! T M N Y ¿ ? 1 L C A ñ : S ú J z P D ” E “ é j f á ó í v h g y q b . , p m c t u d l i r n s o a e - ¨ - False - “ ” - ‘ ’ - “ ” - “ - Yes - ‘ - Yes - “ - No - No - p p/q1 b/q1 - ‌ ‍ _,_ @ _[ ]_ _—_ _- _-_ … _…_ “ ..._ / - _— ” _–_ . —_ …_ — _( )_ ;_ ’_ _‘ !_ _¡ _¿ ?_ :_ ”_ _“ ._ ,_ - _; .._ , ¡ _(_ ! _” - False - 2018-03-06T19:44:35.342721 - 5a9eef95f8b34b5e5a30fc1f - Spanish(spa), El Nuevo Testamento, Versión Biblia Libre, Spain, New Testament, 2018-03-01, spavbl, dbl: matches - - \ No newline at end of file diff --git a/samples/data/VBL-PT/custom.vrs b/samples/data/VBL-PT/custom.vrs deleted file mode 100644 index 8f724428c..000000000 --- a/samples/data/VBL-PT/custom.vrs +++ /dev/null @@ -1,22 +0,0 @@ -# custom.vrs -# Prepared by Michael Johnson. -# amend total quantity of verses in chapter -3JN 1:14 -REV 12:17 -# deliberately missing verses based on textual criticism decisions or alternate versification --MAT 17:21 --MAT 18:11 --MAT 23:14 --MRK 7:16 --MRK 9:44 --MRK 9:46 --MRK 11:26 --MRK 15:28 --LUK 17:36 --LUK 23:17 --JHN 5:4 --ACT 8:37 --ACT 15:34 --ACT 24:7 --ROM 16:24 -#-2CO 13:14 diff --git a/samples/data/VBL-PT/es-Latn.ldml b/samples/data/VBL-PT/es-Latn.ldml deleted file mode 100644 index 7d7d5e2db..000000000 --- a/samples/data/VBL-PT/es-Latn.ldml +++ /dev/null @@ -1,877 +0,0 @@ -[\u00AA\u00BA\u00E0\u00E2-\u00E8\u00EA-\u00EC\u00EE\u00EF\u00F2\u00F4\u00F6\u00F8\u00F9\u00FB\u00FF\u0101\u0103\u0113\u0115\u012B\u012D\u014D\u014F\u0153\u016B\u016D][A-Z\u00D1][!-#\&-*,-/\:;?@\[-\]\u00A1\u00A7\u00AB\u00BB\u00BF\u2010\u2013\u2014\u2018\u2019\u201C\u201D\u2020\u2021\u2026\u2032\u2033][a-z\u00E1\u00E9\u00ED\u00F1\u00F3\u00FA\u00FC][][][][][][]«»“”left-to-rightstandard \ No newline at end of file diff --git a/samples/data/WEB-DBL/license.xml b/samples/data/WEB-DBL/license.xml deleted file mode 100644 index e66fc4032..000000000 --- a/samples/data/WEB-DBL/license.xml +++ /dev/null @@ -1 +0,0 @@ -2015-12-159999-12-31TrueTrueTrueTrue \ No newline at end of file diff --git a/samples/data/WEB-DBL/metadata.xml b/samples/data/WEB-DBL/metadata.xml deleted file mode 100644 index 53880518b..000000000 --- a/samples/data/WEB-DBL/metadata.xml +++ /dev/null @@ -1,139 +0,0 @@ - - - World English Bible (American Edition) - World English Bible - Related dialect adaptation of the American Standard Version of 1901 into modern English - WEB - WEB - Bible with Deuterocanon - 2020-01-01 - Paratext/9.2.102.3 - - 9879dbb7cfe39e4d5d6f7f9dbd0c6414691a036e - 4c4fe70e7e2ad454e20891a60bf81991037976b7 - engWEB14 - World English Bible (American Edition) - - - dtyynkiqGHSk8qEp7 - - - - text - false - true - true - true - Revision - Common - Standard - - - - - eBible.org - http://eBible.org - eBible.org - 545d2cb0ae307afa44b390fc - eBible.org - - - https://eBible.org - 545d2cb0ae307afa44b390fc - eBible.org - - - true - true - true - true - true - 545d2cb0ae307afa44b390fc - eBible.org - - - - eng - English - English - - Latn - LTR - en - 00025 - Arabic - - - - US - United States of America - - - - 3.0 - false - - - - 1 John - 1 John - John’s First Letter - - - 2 John - 2 John - John’s Second Letter - - - 3 John - 3 John - John’s Third Letter - - - - - - - - - - - - - World English Bible - World English Bible - Ecumenical - Ecumenical - WEB - WEB - - - - - - - - - - - - - - - -

PUBLIC DOMAIN

- - - - - -

See http://WorldEnglish.Bible

- - - - Kahunapule Michael Johnson - 2021-12-10T23:21:07.923054 - 2022-01-14T03:08:53.100483 - Added missing "to" to EXO 13:11 - - \ No newline at end of file diff --git a/samples/data/WEB-DBL/release/USX_1/1JN.usx b/samples/data/WEB-DBL/release/USX_1/1JN.usx deleted file mode 100644 index afb572261..000000000 --- a/samples/data/WEB-DBL/release/USX_1/1JN.usx +++ /dev/null @@ -1,107 +0,0 @@ - - - 62-1JN-web.sfm World English Bible (WEB) - UTF-8 - 1 John - John’s First Letter - 1 John - 1 John - John’s First Letter - - - - That which was from the beginning, that which we have heard, that which we have seen with our eyes, that which we saw, and our hands touched, concerning the Word of life (and the life was revealed, and we have seen, and testify, and declare to you the life, the eternal life, which was with the Father, and was revealed to us); that which we have seen and heard we declare to you, that you also may have fellowship with us. Yes, and our fellowship is with the Father and with his Son, Jesus Christ.1:3 “Christ” means “Anointed One”. And we write these things to you, that our joy may be fulfilled. - - - This is the message which we have heard from him and announce to you, that God is light, and in him is no darkness at all. If we say that we have fellowship with him and walk in the darkness, we lie and don’t tell the truth. But if we walk in the light as he is in the light, we have fellowship with one another, and the blood of Jesus Christ his Son, cleanses us from all sin. If we say that we have no sin, we deceive ourselves, and the truth is not in us. If we confess our sins, he is faithful and righteous to forgive us the sins and to cleanse us from all unrighteousness. If we say that we haven’t sinned, we make him a liar, and his word is not in us. - - - - - My little children, I write these things to you so that you may not sin. If anyone sins, we have a Counselor2:1 Greek παρακλητον: Counselor, Helper, Intercessor, Advocate, and Comforter. with the Father, Jesus Christ, the righteous. And he is the atoning sacrifice2:2 “atoning sacrifice” is from the Greek “ιλασμος”, an appeasing, propitiating, or the means of appeasement or propitiation—the sacrifice that turns away God’s wrath because of our sin. for our sins, and not for ours only, but also for the whole world. This is how we know that we know him: if we keep his commandments. One who says, “I know him,” and doesn’t keep his commandments, is a liar, and the truth isn’t in him. But God’s love has most certainly been perfected in whoever keeps his word. This is how we know that we are in him: he who says he remains in him ought himself also to walk just like he walked. - - Brothers, I write no new commandment to you, but an old commandment which you had from the beginning. The old commandment is the word which you heard from the beginning. Again, I write a new commandment to you, which is true in him and in you, because the darkness is passing away and the true light already shines. He who says he is in the light and hates his brother is in the darkness even until now. He who loves his brother remains in the light, and there is no occasion for stumbling in him. But he who hates his brother is in the darkness, and walks in the darkness, and doesn’t know where he is going, because the darkness has blinded his eyes. - - - I write to you, little children, because your sins are forgiven you for his name’s sake. - - - I write to you, fathers, because you know him who is from the beginning. - - I write to you, young men, because you have overcome the evil one. - - I write to you, little children, because you know the Father. - - - I have written to you, fathers, because you know him who is from the beginning. - - I have written to you, young men, because you are strong, and the word of God remains in you, and you have overcome the evil one. - - - Don’t love the world or the things that are in the world. If anyone loves the world, the Father’s love isn’t in him. For all that is in the world—the lust of the flesh, the lust of the eyes, and the pride of life—isn’t the Father’s, but is the world’s. The world is passing away with its lusts, but he who does God’s will remains forever. - - Little children, these are the end times, and as you heard that the Antichrist is coming, even now many antichrists have arisen. By this we know that it is the final hour. They went out from us, but they didn’t belong to us; for if they had belonged to us, they would have continued with us. But they left, that they might be revealed that none of them belong to us. You have an anointing from the Holy One, and you all have knowledge.2:20 Or, “know what is true”, or, “know all things” I have not written to you because you don’t know the truth, but because you know it, and because no lie is of the truth. Who is the liar but he who denies that Jesus is the Christ? This is the Antichrist, he who denies the Father and the Son. Whoever denies the Son doesn’t have the Father. He who confesses the Son has the Father also. - - Therefore, as for you, let that remain in you which you heard from the beginning. If that which you heard from the beginning remains in you, you also will remain in the Son, and in the Father. This is the promise which he promised us, the eternal life. - - - These things I have written to you concerning those who would lead you astray. As for you, the anointing which you received from him remains in you, and you don’t need for anyone to teach you. But as his anointing teaches you concerning all things, and is true, and is no lie, and even as it taught you, you will remain in him. - - - Now, little children, remain in him, that when he appears, we may have boldness and not be ashamed before him at his coming. If you know that he is righteous, you know that everyone who practices righteousness has been born of him. - - - - - See how great a love the Father has given to us, that we should be called children of God! For this cause the world doesn’t know us, because it didn’t know him. Beloved, now we are children of God. It is not yet revealed what we will be; but we know that when he is revealed, we will be like him, for we will see him just as he is. Everyone who has this hope set on him purifies himself, even as he is pure. - - - Everyone who sins also commits lawlessness. Sin is lawlessness. You know that he was revealed to take away our sins, and no sin is in him. Whoever remains in him doesn’t sin. Whoever sins hasn’t seen him and doesn’t know him. - - - Little children, let no one lead you astray. He who does righteousness is righteous, even as he is righteous. He who sins is of the devil, for the devil has been sinning from the beginning. To this end the Son of God was revealed: that he might destroy the works of the devil. Whoever is born of God doesn’t commit sin, because his seed remains in him, and he can’t sin, because he is born of God. In this the children of God are revealed, and the children of the devil. Whoever doesn’t do righteousness is not of God, neither is he who doesn’t love his brother. For this is the message which you heard from the beginning, that we should love one another— unlike Cain, who was of the evil one and killed his brother. Why did he kill him? Because his deeds were evil, and his brother’s righteous. - - - Don’t be surprised, my brothers, if the world hates you. We know that we have passed out of death into life, because we love the brothers. He who doesn’t love his brother remains in death. Whoever hates his brother is a murderer, and you know that no murderer has eternal life remaining in him. - - - By this we know love, because he laid down his life for us. And we ought to lay down our lives for the brothers. But whoever has the world’s goods and sees his brother in need, then closes his heart of compassion against him, how does God’s love remain in him? - - - My little children, let’s not love in word only, or with the tongue only, but in deed and truth. And by this we know that we are of the truth and persuade our hearts before him, because if our heart condemns us, God is greater than our heart, and knows all things. Beloved, if our hearts don’t condemn us, we have boldness toward God; so whatever we ask, we receive from him, because we keep his commandments and do the things that are pleasing in his sight. This is his commandment, that we should believe in the name of his Son, Jesus Christ, and love one another, even as he commanded. He who keeps his commandments remains in him, and he in him. By this we know that he remains in us, by the Spirit which he gave us. - - - - - Beloved, don’t believe every spirit, but test the spirits, whether they are of God, because many false prophets have gone out into the world. By this you know the Spirit of God: every spirit who confesses that Jesus Christ has come in the flesh is of God, and every spirit who doesn’t confess that Jesus Christ has come in the flesh is not of God; and this is the spirit of the Antichrist, of whom you have heard that it comes. Now it is in the world already. You are of God, little children, and have overcome them, because greater is he who is in you than he who is in the world. They are of the world. Therefore they speak of the world, and the world hears them. We are of God. He who knows God listens to us. He who is not of God doesn’t listen to us. By this we know the spirit of truth, and the spirit of error. - - - Beloved, let’s love one another, for love is of God; and everyone who loves has been born of God and knows God. He who doesn’t love doesn’t know God, for God is love. By this God’s love was revealed in us, that God has sent his only born4:9 The phrase “only born” is from the Greek word “μονογενη”, which is sometimes translated “only begotten” or “one and only”. Son into the world that we might live through him. In this is love, not that we loved God, but that he loved us, and sent his Son as the atoning sacrifice4:10 “atoning sacrifice” is from the Greek “ιλασμος”, an appeasing, propitiating, or the means of appeasement or propitiation—the sacrifice that turns away God’s wrath because of our sin. for our sins. Beloved, if God loved us in this way, we also ought to love one another. No one has seen God at any time. If we love one another, God remains in us, and his love has been perfected in us. - - - By this we know that we remain in him and he in us, because he has given us of his Spirit. We have seen and testify that the Father has sent the Son as the Savior of the world. Whoever confesses that Jesus is the Son of God, God remains in him, and he in God. We know and have believed the love which God has for us. God is love, and he who remains in love remains in God, and God remains in him. In this, love has been made perfect among us, that we may have boldness in the day of judgment, because as he is, even so we are in this world. There is no fear in love; but perfect love casts out fear, because fear has punishment. He who fears is not made perfect in love. We love him,4:19 NU omits “him”. because he first loved us. If a man says, “I love God,” and hates his brother, he is a liar; for he who doesn’t love his brother whom he has seen, how can he love God whom he has not seen? This commandment we have from him, that he who loves God should also love his brother. - - - - - Whoever believes that Jesus is the Christ has been born of God. Whoever loves the Father also loves the child who is born of him. By this we know that we love the children of God, when we love God and keep his commandments. For this is loving God, that we keep his commandments. His commandments are not grievous. For whatever is born of God overcomes the world. This is the victory that has overcome the world: your faith. Who is he who overcomes the world, but he who believes that Jesus is the Son of God? - - - This is he who came by water and blood, Jesus Christ; not with the water only, but with the water and the blood. It is the Spirit who testifies, because the Spirit is the truth. For there are three who testify:5:7 Only a few recent manuscripts add “in heaven: the Father, the Word, and the Holy Spirit; and these three are one. And there are three that testify on earth:” the Spirit, the water, and the blood; and the three agree as one. If we receive the witness of men, the witness of God is greater; for this is God’s testimony which he has testified concerning his Son. He who believes in the Son of God has the testimony in himself. He who doesn’t believe God has made him a liar, because he has not believed in the testimony that God has given concerning his Son. The testimony is this: that God gave to us eternal life, and this life is in his Son. He who has the Son has the life. He who doesn’t have God’s Son doesn’t have the life. - - - These things I have written to you who believe in the name of the Son of God, that you may know that you have eternal life, and that you may continue to believe in the name of the Son of God. - - - This is the boldness which we have toward him, that if we ask anything according to his will, he listens to us. And if we know that he listens to us, whatever we ask, we know that we have the petitions which we have asked of him. - - - If anyone sees his brother sinning a sin not leading to death, he shall ask, and God will give him life for those who sin not leading to death. There is sin leading to death. I don’t say that he should make a request concerning this. All unrighteousness is sin, and there is sin not leading to death. - - - We know that whoever is born of God doesn’t sin, but he who was born of God keeps himself, and the evil one doesn’t touch him. We know that we are of God, and the whole world lies in the power of the evil one. We know that the Son of God has come and has given us an understanding, that we know him who is true; and we are in him who is true, in his Son Jesus Christ. This is the true God and eternal life. - - - Little children, keep yourselves from idols. - - \ No newline at end of file diff --git a/samples/data/WEB-DBL/release/USX_1/2JN.usx b/samples/data/WEB-DBL/release/USX_1/2JN.usx deleted file mode 100644 index 4e17914ee..000000000 --- a/samples/data/WEB-DBL/release/USX_1/2JN.usx +++ /dev/null @@ -1,24 +0,0 @@ - - - 63-2JN-web.sfm World English Bible (WEB) - UTF-8 - 2 John - John’s Second Letter - 2 John - 2 John - John’s Second Letter - - - - The elder, to the chosen lady and her children, whom I love in truth, and not I only, but also all those who know the truth, for the truth’s sake, which remains in us, and it will be with us forever: Grace, mercy, and peace will be with us, from God the Father and from the Lord Jesus Christ,1:3 “Christ” means “Anointed One”. the Son of the Father, in truth and love. - - - I rejoice greatly that I have found some of your children walking in truth, even as we have been commanded by the Father. Now I beg you, dear lady, not as though I wrote to you a new commandment, but that which we had from the beginning, that we love one another. This is love, that we should walk according to his commandments. This is the commandment, even as you heard from the beginning, that you should walk in it. - - - For many deceivers have gone out into the world, those who don’t confess that Jesus Christ came in the flesh. This is the deceiver and the Antichrist. Watch yourselves, that we don’t lose the things which we have accomplished, but that we receive a full reward. Whoever transgresses and doesn’t remain in the teaching of Christ doesn’t have God. He who remains in the teaching has both the Father and the Son. If anyone comes to you and doesn’t bring this teaching, don’t receive him into your house, and don’t welcome him, for he who welcomes him participates in his evil deeds. - - - Having many things to write to you, I don’t want to do so with paper and ink, but I hope to come to you and to speak face to face, that our joy may be made full. The children of your chosen sister greet you. Amen. - - \ No newline at end of file diff --git a/samples/data/WEB-DBL/release/USX_1/3JN.usx b/samples/data/WEB-DBL/release/USX_1/3JN.usx deleted file mode 100644 index 02f756613..000000000 --- a/samples/data/WEB-DBL/release/USX_1/3JN.usx +++ /dev/null @@ -1,31 +0,0 @@ - - - 64-3JN-web.sfm World English Bible (WEB) - UTF-8 - 3 John - John’s Third Letter - 3 John - 3 John - John’s Third Letter - - - - The elder to Gaius the beloved, whom I love in truth. - - - Beloved, I pray that you may prosper in all things and be healthy, even as your soul prospers. For I rejoiced greatly when brothers came and testified about your truth, even as you walk in truth. I have no greater joy than this: to hear about my children walking in truth. - - - Beloved, you do a faithful work in whatever you accomplish for those who are brothers and strangers. They have testified about your love before the assembly. You will do well to send them forward on their journey in a way worthy of God, because for the sake of the Name they went out, taking nothing from the Gentiles. We therefore ought to receive such, that we may be fellow workers for the truth. - - - I wrote to the assembly, but Diotrephes, who loves to be first among them, doesn’t accept what we say. Therefore, if I come, I will call attention to his deeds which he does, unjustly accusing us with wicked words. Not content with this, he doesn’t receive the brothers himself, and those who would, he forbids and throws out of the assembly. - - - Beloved, don’t imitate that which is evil, but that which is good. He who does good is of God. He who does evil hasn’t seen God. Demetrius has the testimony of all, and of the truth itself; yes, we also testify, and you know that our testimony is true. - - - I had many things to write to you, but I am unwilling to write to you with ink and pen; but I hope to see you soon. Then we will speak face to face. - Peace be to you. The friends greet you. Greet the friends by name. - - \ No newline at end of file diff --git a/samples/data/WEB-DBL/release/eng_en.ldml b/samples/data/WEB-DBL/release/eng_en.ldml deleted file mode 100644 index 1371f1ba8..000000000 --- a/samples/data/WEB-DBL/release/eng_en.ldml +++ /dev/null @@ -1 +0,0 @@ -[][!%-/\:;=?\[\]\u00B0\u00D7\u200C\u200D\u2013\u2014\u2018\u2019\u201C\u201D\u2303{...}][][\ '\-\u2019][][][a b c d e f g h i j k l m n o p q r s t u v w x y z {aa} {bb} {cc} {dd} {ee} {ff} {gg} {hh} {ii} {jj} {kk} {ll} {mm} {nn} {oo} {pp} {qq} {rr} {ss} {tt} {uu} {vv} {ww} {xx} {yy} {zz}][]“”‘’left-to-rightstandard \ No newline at end of file diff --git a/samples/data/WEB-DBL/release/styles.xml b/samples/data/WEB-DBL/release/styles.xml deleted file mode 100644 index 6614341eb..000000000 --- a/samples/data/WEB-DBL/release/styles.xml +++ /dev/null @@ -1,1817 +0,0 @@ - - - Gentium Plus - 14 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/samples/data/WEB-DBL/release/versification.vrs b/samples/data/WEB-DBL/release/versification.vrs deleted file mode 100644 index 19d8f1511..000000000 --- a/samples/data/WEB-DBL/release/versification.vrs +++ /dev/null @@ -1,395 +0,0 @@ -# List of books, chapters, verses -# One line per book. -# One entry for each chapter. -# Verse number is the maximum verse number for that chapter. -GEN 1:31 2:25 3:24 4:26 5:32 6:22 7:24 8:22 9:29 10:32 11:32 12:20 13:18 14:24 15:21 16:16 17:27 18:33 19:38 20:18 21:34 22:24 23:20 24:67 25:34 26:35 27:46 28:22 29:35 30:43 31:55 32:32 33:20 34:31 35:29 36:43 37:36 38:30 39:23 40:23 41:57 42:38 43:34 44:34 45:28 46:34 47:31 48:22 49:33 50:26 -EXO 1:22 2:25 3:22 4:31 5:23 6:30 7:25 8:32 9:35 10:29 11:10 12:51 13:22 14:31 15:27 16:36 17:16 18:27 19:25 20:26 21:36 22:31 23:33 24:18 25:40 26:37 27:21 28:43 29:46 30:38 31:18 32:35 33:23 34:35 35:35 36:38 37:29 38:31 39:43 40:38 -LEV 1:17 2:16 3:17 4:35 5:19 6:30 7:38 8:36 9:24 10:20 11:47 12:8 13:59 14:57 15:33 16:34 17:16 18:30 19:37 20:27 21:24 22:33 23:44 24:23 25:55 26:46 27:34 -NUM 1:54 2:34 3:51 4:49 5:31 6:27 7:89 8:26 9:23 10:36 11:35 12:16 13:33 14:45 15:41 16:50 17:13 18:32 19:22 20:29 21:35 22:41 23:30 24:25 25:18 26:65 27:23 28:31 29:40 30:16 31:54 32:42 33:56 34:29 35:34 36:13 -DEU 1:46 2:37 3:29 4:49 5:33 6:25 7:26 8:20 9:29 10:22 11:32 12:32 13:18 14:29 15:23 16:22 17:20 18:22 19:21 20:20 21:23 22:30 23:25 24:22 25:19 26:19 27:26 28:68 29:29 30:20 31:30 32:52 33:29 34:12 -JOS 1:18 2:24 3:17 4:24 5:15 6:27 7:26 8:35 9:27 10:43 11:23 12:24 13:33 14:15 15:63 16:10 17:18 18:28 19:51 20:9 21:45 22:34 23:16 24:33 -JDG 1:36 2:23 3:31 4:24 5:31 6:40 7:25 8:35 9:57 10:18 11:40 12:15 13:25 14:20 15:20 16:31 17:13 18:31 19:30 20:48 21:25 -RUT 1:22 2:23 3:18 4:22 -1SA 1:28 2:36 3:21 4:22 5:12 6:21 7:17 8:22 9:27 10:27 11:15 12:25 13:23 14:52 15:35 16:23 17:58 18:30 19:24 20:42 21:15 22:23 23:29 24:22 25:44 26:25 27:12 28:25 29:11 30:31 31:13 -2SA 1:27 2:32 3:39 4:12 5:25 6:23 7:29 8:18 9:13 10:19 11:27 12:31 13:39 14:33 15:37 16:23 17:29 18:33 19:43 20:26 21:22 22:51 23:39 24:25 -1KI 1:53 2:46 3:28 4:34 5:18 6:38 7:51 8:66 9:28 10:29 11:43 12:33 13:34 14:31 15:34 16:34 17:24 18:46 19:21 20:43 21:29 22:53 -2KI 1:18 2:25 3:27 4:44 5:27 6:33 7:20 8:29 9:37 10:36 11:21 12:21 13:25 14:29 15:38 16:20 17:41 18:37 19:37 20:21 21:26 22:20 23:37 24:20 25:30 -1CH 1:54 2:55 3:24 4:43 5:26 6:81 7:40 8:40 9:44 10:14 11:47 12:40 13:14 14:17 15:29 16:43 17:27 18:17 19:19 20:8 21:30 22:19 23:32 24:31 25:31 26:32 27:34 28:21 29:30 -2CH 1:17 2:18 3:17 4:22 5:14 6:42 7:22 8:18 9:31 10:19 11:23 12:16 13:22 14:15 15:19 16:14 17:19 18:34 19:11 20:37 21:20 22:12 23:21 24:27 25:28 26:23 27:9 28:27 29:36 30:27 31:21 32:33 33:25 34:33 35:27 36:23 -EZR 1:11 2:70 3:13 4:24 5:17 6:22 7:28 8:36 9:15 10:44 -NEH 1:11 2:20 3:32 4:23 5:19 6:19 7:73 8:18 9:38 10:39 11:36 12:47 13:31 -EST 1:22 2:23 3:15 4:17 5:14 6:14 7:10 8:17 9:32 10:3 -JOB 1:22 2:13 3:26 4:21 5:27 6:30 7:21 8:22 9:35 10:22 11:20 12:25 13:28 14:22 15:35 16:22 17:16 18:21 19:29 20:29 21:34 22:30 23:17 24:25 25:6 26:14 27:23 28:28 29:25 30:31 31:40 32:22 33:33 34:37 35:16 36:33 37:24 38:41 39:30 40:24 41:34 42:17 -PSA 1:6 2:12 3:8 4:8 5:12 6:10 7:17 8:9 9:20 10:18 11:7 12:8 13:6 14:7 15:5 16:11 17:15 18:50 19:14 20:9 21:13 22:31 23:6 24:10 25:22 26:12 27:14 28:9 29:11 30:12 31:24 32:11 33:22 34:22 35:28 36:12 37:40 38:22 39:13 40:17 41:13 42:11 43:5 44:26 45:17 46:11 47:9 48:14 49:20 50:23 51:19 52:9 53:6 54:7 55:23 56:13 57:11 58:11 59:17 60:12 61:8 62:12 63:11 64:10 65:13 66:20 67:7 68:35 69:36 70:5 71:24 72:20 73:28 74:23 75:10 76:12 77:20 78:72 79:13 80:19 81:16 82:8 83:18 84:12 85:13 86:17 87:7 88:18 89:52 90:17 91:16 92:15 93:5 94:23 95:11 96:13 97:12 98:9 99:9 100:5 101:8 102:28 103:22 104:35 105:45 106:48 107:43 108:13 109:31 110:7 111:10 112:10 113:9 114:8 115:18 116:19 117:2 118:29 119:176 120:7 121:8 122:9 123:4 124:8 125:5 126:6 127:5 128:6 129:8 130:8 131:3 132:18 133:3 134:3 135:21 136:26 137:9 138:8 139:24 140:13 141:10 142:7 143:12 144:15 145:21 146:10 147:20 148:14 149:9 150:6 -PRO 1:33 2:22 3:35 4:27 5:23 6:35 7:27 8:36 9:18 10:32 11:31 12:28 13:25 14:35 15:33 16:33 17:28 18:24 19:29 20:30 21:31 22:29 23:35 24:34 25:28 26:28 27:27 28:28 29:27 30:33 31:31 -ECC 1:18 2:26 3:22 4:16 5:20 6:12 7:29 8:17 9:18 10:20 11:10 12:14 -SNG 1:17 2:17 3:11 4:16 5:16 6:13 7:13 8:14 -ISA 1:31 2:22 3:26 4:6 5:30 6:13 7:25 8:22 9:21 10:34 11:16 12:6 13:22 14:32 15:9 16:14 17:14 18:7 19:25 20:6 21:17 22:25 23:18 24:23 25:12 26:21 27:13 28:29 29:24 30:33 31:9 32:20 33:24 34:17 35:10 36:22 37:38 38:22 39:8 40:31 41:29 42:25 43:28 44:28 45:25 46:13 47:15 48:22 49:26 50:11 51:23 52:15 53:12 54:17 55:13 56:12 57:21 58:14 59:21 60:22 61:11 62:12 63:19 64:12 65:25 66:24 -JER 1:19 2:37 3:25 4:31 5:31 6:30 7:34 8:22 9:26 10:25 11:23 12:17 13:27 14:22 15:21 16:21 17:27 18:23 19:15 20:18 21:14 22:30 23:40 24:10 25:38 26:24 27:22 28:17 29:32 30:24 31:40 32:44 33:26 34:22 35:19 36:32 37:21 38:28 39:18 40:16 41:18 42:22 43:13 44:30 45:5 46:28 47:7 48:47 49:39 50:46 51:64 52:34 -LAM 1:22 2:22 3:66 4:22 5:22 -EZK 1:28 2:10 3:27 4:17 5:17 6:14 7:27 8:18 9:11 10:22 11:25 12:28 13:23 14:23 15:8 16:63 17:24 18:32 19:14 20:49 21:32 22:31 23:49 24:27 25:17 26:21 27:36 28:26 29:21 30:26 31:18 32:32 33:33 34:31 35:15 36:38 37:28 38:23 39:29 40:49 41:26 42:20 43:27 44:31 45:25 46:24 47:23 48:35 -DAN 1:21 2:49 3:30 4:37 5:31 6:28 7:28 8:27 9:27 10:21 11:45 12:13 -HOS 1:11 2:23 3:5 4:19 5:15 6:11 7:16 8:14 9:17 10:15 11:12 12:14 13:16 14:9 -JOL 1:20 2:32 3:21 -AMO 1:15 2:16 3:15 4:13 5:27 6:14 7:17 8:14 9:15 -OBA 1:21 -JON 1:17 2:10 3:10 4:11 -MIC 1:16 2:13 3:12 4:13 5:15 6:16 7:20 -NAM 1:15 2:13 3:19 -HAB 1:17 2:20 3:19 -ZEP 1:18 2:15 3:20 -HAG 1:15 2:23 -ZEC 1:21 2:13 3:10 4:14 5:11 6:15 7:14 8:23 9:17 10:12 11:17 12:14 13:9 14:21 -MAL 1:14 2:17 3:18 4:6 -MAT 1:25 2:23 3:17 4:25 5:48 6:34 7:29 8:34 9:38 10:42 11:30 12:50 13:58 14:36 15:39 16:28 17:27 18:35 19:30 20:34 21:46 22:46 23:39 24:51 25:46 26:75 27:66 28:20 -MRK 1:45 2:28 3:35 4:41 5:43 6:56 7:37 8:38 9:50 10:52 11:33 12:44 13:37 14:72 15:47 16:20 -LUK 1:80 2:52 3:38 4:44 5:39 6:49 7:50 8:56 9:62 10:42 11:54 12:59 13:35 14:35 15:32 16:31 17:37 18:43 19:48 20:47 21:38 22:71 23:56 24:53 -JHN 1:51 2:25 3:36 4:54 5:47 6:71 7:53 8:59 9:41 10:42 11:57 12:50 13:38 14:31 15:27 16:33 17:26 18:40 19:42 20:31 21:25 -ACT 1:26 2:47 3:26 4:37 5:42 6:15 7:60 8:40 9:43 10:48 11:30 12:25 13:52 14:28 15:41 16:40 17:34 18:28 19:41 20:38 21:40 22:30 23:35 24:27 25:27 26:32 27:44 28:31 -ROM 1:32 2:29 3:31 4:25 5:21 6:23 7:25 8:39 9:33 10:21 11:36 12:21 13:14 14:26 15:33 16:27 -1CO 1:31 2:16 3:23 4:21 5:13 6:20 7:40 8:13 9:27 10:33 11:34 12:31 13:13 14:40 15:58 16:24 -2CO 1:24 2:17 3:18 4:18 5:21 6:18 7:16 8:24 9:15 10:18 11:33 12:21 13:14 -GAL 1:24 2:21 3:29 4:31 5:26 6:18 -EPH 1:23 2:22 3:21 4:32 5:33 6:24 -PHP 1:30 2:30 3:21 4:23 -COL 1:29 2:23 3:25 4:18 -1TH 1:10 2:20 3:13 4:18 5:28 -2TH 1:12 2:17 3:18 -1TI 1:20 2:15 3:16 4:16 5:25 6:21 -2TI 1:18 2:26 3:17 4:22 -TIT 1:16 2:15 3:15 -PHM 1:25 -HEB 1:14 2:18 3:19 4:16 5:14 6:20 7:28 8:13 9:28 10:39 11:40 12:29 13:25 -JAS 1:27 2:26 3:18 4:17 5:20 -1PE 1:25 2:25 3:22 4:19 5:14 -2PE 1:21 2:22 3:18 -1JN 1:10 2:29 3:24 4:21 5:21 -2JN 1:13 -3JN 1:15 -JUD 1:25 -REV 1:20 2:29 3:22 4:11 5:14 6:17 7:17 8:13 9:21 10:11 11:19 12:17 13:18 14:20 15:8 16:21 17:18 18:24 19:21 20:15 21:27 22:21 -TOB 1:22 2:14 3:17 4:21 5:22 6:17 7:18 8:21 9:6 10:12 11:19 12:22 13:18 14:15 -JDT 1:16 2:28 3:10 4:15 5:24 6:21 7:32 8:36 9:14 10:23 11:23 12:20 13:20 14:19 15:13 16:25 -ESG 1:22 2:23 3:15 4:47 5:14 6:14 7:10 8:17 9:32 10:14 -WIS 1:16 2:24 3:19 4:20 5:23 6:25 7:30 8:21 9:18 10:21 11:26 12:27 13:19 14:31 15:19 16:29 17:21 18:25 19:22 -SIR 1:30 2:18 3:31 4:31 5:15 6:37 7:36 8:19 9:18 10:31 11:34 12:18 13:26 14:27 15:20 16:30 17:32 18:33 19:30 20:32 21:28 22:27 23:28 24:34 25:26 26:29 27:30 28:26 29:28 30:25 31:31 32:24 33:33 34:26 35:20 36:26 37:31 38:34 39:35 40:30 41:24 42:25 43:33 44:23 45:26 46:20 47:25 48:25 49:16 50:29 51:30 -BAR 1:22 2:35 3:37 4:37 5:9 6:73 -LJE 1:73 -S3Y 1:68 -SUS 1:64 -BEL 1:42 -1MA 1:64 2:70 3:60 4:61 5:68 6:63 7:50 8:32 9:73 10:89 11:74 12:53 13:53 14:49 15:41 16:24 -2MA 1:36 2:32 3:40 4:50 5:27 6:31 7:42 8:36 9:29 10:38 11:38 12:45 13:26 14:46 15:39 -3MA 1:29 2:33 3:30 4:21 5:51 6:41 7:23 -4MA 1:35 2:24 3:21 4:26 5:38 6:35 7:25 8:29 9:32 10:21 11:27 12:20 13:27 14:20 15:32 16:25 17:24 18:24 -1ES 1:58 2:30 3:24 4:63 5:73 6:34 7:15 8:96 9:55 -2ES 1:40 2:48 3:36 4:52 5:56 6:59 7:140 8:63 9:47 10:59 11:46 12:51 13:58 14:48 15:63 16:78 -MAN 1:15 -PS2 1:7 -ODA 1:1 -PSS 1:1 -JSA 1:18 2:24 3:17 4:24 5:15 6:27 7:26 8:35 9:27 10:43 11:23 12:24 13:33 14:15 15:63 16:10 17:18 18:28 19:51 20:9 21:45 22:34 23:16 24:33 -JDB 1:36 2:23 3:31 4:24 5:31 6:40 7:25 8:35 9:57 10:18 11:40 12:15 13:25 14:20 15:20 16:31 17:13 18:31 19:30 20:48 21:25 -TBS 1:22 2:14 3:17 4:21 5:23 6:19 7:17 8:21 9:6 10:14 11:19 12:22 13:18 14:15 -SST 1:64 -DNT 1:21 2:49 3:97 4:37 5:30 6:29 7:28 8:27 9:27 10:21 11:45 12:13 -BLT 1:42 -XXA 1:1 -XXB 1:1 -XXC 1:1 -XXD 1:1 -XXE 1:1 -XXF 1:1 -XXG 1:1 -FRT 1:1 -BAK 1:1 -OTH 1:1 -3ES 1:1 -EZA 1:1 -5EZ 1:1 -6EZ 1:1 -INT 1:1 -CNC 1:1 -GLO 1:1 -TDX 1:1 -NDX 1:1 -DAG 1:21 2:49 3:97 4:37 5:31 6:28 7:28 8:27 9:27 10:21 11:45 12:13 13:64 14:42 -PS3 1:1 -2BA 1:1 -LBA 1:1 -JUB 1:1 -ENO 1:1 -1MQ 1:1 -2MQ 1:1 -3MQ 1:1 -REP 1:1 -4BA 1:1 -LAO 1:20 -# -# Mappings from this versification to standard versification -GEN 31:55 = GEN 32:1 -GEN 32:1-32 = GEN 32:2-33 -EXO 8:1-4 = EXO 7:26-29 -EXO 8:5-32 = EXO 8:1-28 -EXO 22:1 = EXO 21:37 -EXO 22:2-31 = EXO 22:1-30 -LEV 6:1-7 = LEV 5:20-26 -LEV 6:8-30 = LEV 6:1-23 -NUM 16:36-50 = NUM 17:1-15 -NUM 17:1-13 = NUM 17:16-28 -NUM 29:40 = NUM 30:1 -NUM 30:1-16 = NUM 30:2-17 -DEU 12:32 = DEU 13:1 -DEU 13:1-18 = DEU 13:2-19 -DEU 22:30 = DEU 23:1 -DEU 23:1-25 = DEU 23:2-26 -DEU 29:1 = DEU 28:69 -DEU 29:2-29 = DEU 29:1-28 -1SA 20:42 = 1SA 21:1 -1SA 21:1-15 = 1SA 21:2-16 -1SA 23:29 = 1SA 24:1 -1SA 24:1-22 = 1SA 24:2-23 -2SA 18:33 = 2SA 19:1 -2SA 19:1-43 = 2SA 19:2-44 -1KI 4:21-34 = 1KI 5:1-14 -1KI 5:1-18 = 1KI 5:15-32 -1KI 22:43-53 = 1KI 22:44-54 -2KI 11:21 = 2KI 12:1 -2KI 12:1-21 = 2KI 12:2-22 -1CH 6:1-15 = 1CH 5:27-41 -1CH 6:16-81 = 1CH 6:1-66 -1CH 12:4-40 = 1CH 12:5-41 -2CH 2:1 = 2CH 1:18 -2CH 2:2-18 = 2CH 2:1-17 -2CH 14:1 = 2CH 13:23 -2CH 14:2-15 = 2CH 14:1-14 -NEH 4:1-6 = NEH 3:33-38 -NEH 4:7-23 = NEH 4:1-17 -NEH 7:69-73 = NEH 7:68-72 -NEH 9:38 = NEH 10:1 -NEH 10:1-39 = NEH 10:2-40 -JOB 41:1-8 = JOB 40:25-32 -JOB 41:9-34 = JOB 41:1-26 -PSA 3:0-8 = PSA 3:1-9 -PSA 4:0-8 = PSA 4:1-9 -PSA 5:0-12 = PSA 5:1-13 -PSA 6:0-10 = PSA 6:1-11 -PSA 7:0-17 = PSA 7:1-18 -PSA 8:0-9 = PSA 8:1-10 -PSA 9:0-20 = PSA 9:1-21 -PSA 12:0-8 = PSA 12:1-9 -PSA 13:0-5 = PSA 13:1-6 -PSA 18:0-50 = PSA 18:1-51 -PSA 19:0-14 = PSA 19:1-15 -PSA 20:0-9 = PSA 20:1-10 -PSA 21:0-13 = PSA 21:1-14 -PSA 22:0-31 = PSA 22:1-32 -PSA 30:0-12 = PSA 30:1-13 -PSA 31:0-24 = PSA 31:1-25 -PSA 34:0-22 = PSA 34:1-23 -PSA 36:0-12 = PSA 36:1-13 -PSA 38:0-22 = PSA 38:1-23 -PSA 39:0-13 = PSA 39:1-14 -PSA 40:0-17 = PSA 40:1-18 -PSA 41:0-13 = PSA 41:1-14 -PSA 42:0-11 = PSA 42:1-12 -PSA 44:0-26 = PSA 44:1-27 -PSA 45:0-17 = PSA 45:1-18 -PSA 46:0-11 = PSA 46:1-12 -PSA 47:0-9 = PSA 47:1-10 -PSA 48:0-14 = PSA 48:1-15 -PSA 49:0-20 = PSA 49:1-21 -PSA 51:0-19 = PSA 51:2-21 -PSA 52:0-9 = PSA 52:2-11 -PSA 53:0-6 = PSA 53:1-7 -PSA 54:0-7 = PSA 54:2-9 -PSA 55:0-23 = PSA 55:1-24 -PSA 56:0-13 = PSA 56:1-14 -PSA 57:0-11 = PSA 57:1-12 -PSA 58:0-11 = PSA 58:1-12 -PSA 59:0-17 = PSA 59:1-18 -PSA 60:0-12 = PSA 60:2-14 -PSA 61:0-8 = PSA 61:1-9 -PSA 62:0-12 = PSA 62:1-13 -PSA 63:0-11 = PSA 63:1-12 -PSA 64:0-10 = PSA 64:1-11 -PSA 65:0-13 = PSA 65:1-14 -PSA 67:0-7 = PSA 67:1-8 -PSA 68:0-35 = PSA 68:1-36 -PSA 69:0-36 = PSA 69:1-37 -PSA 70:0-5 = PSA 70:1-6 -PSA 75:0-10 = PSA 75:1-11 -PSA 76:0-12 = PSA 76:1-13 -PSA 77:0-20 = PSA 77:1-21 -PSA 80:0-19 = PSA 80:1-20 -PSA 81:0-16 = PSA 81:1-17 -PSA 83:0-18 = PSA 83:1-19 -PSA 84:0-12 = PSA 84:1-13 -PSA 85:0-13 = PSA 85:1-14 -PSA 88:0-18 = PSA 88:1-19 -PSA 89:0-52 = PSA 89:1-53 -PSA 92:0-15 = PSA 92:1-16 -PSA 102:0-28 = PSA 102:1-29 -PSA 108:0-13 = PSA 108:1-14 -PSA 140:0-13 = PSA 140:1-14 -PSA 142:0-7 = PSA 142:1-8 -ECC 5:1 = ECC 4:17 -ECC 5:2-20 = ECC 5:1-19 -SNG 6:13 = SNG 7:1 -SNG 7:1-13 = SNG 7:2-14 -ISA 9:1 = ISA 8:23 -ISA 9:2-21 = ISA 9:1-20 -ISA 64:2-12 = ISA 64:1-11 -JER 9:1 = JER 8:23 -JER 9:2-26 = JER 9:1-25 -EZK 20:45-49 = EZK 21:1-5 -EZK 21:1-32 = EZK 21:6-37 -DAN 4:1-3 = DAN 3:31-33 -DAN 4:4-37 = DAN 4:1-34 -DAN 5:31 = DAN 6:1 -DAN 6:1-28 = DAN 6:2-29 -HOS 1:10-11 = HOS 2:1-2 -HOS 2:1-23 = HOS 2:3-25 -HOS 11:12 = HOS 12:1 -HOS 12:1-14 = HOS 12:2-15 -HOS 13:16 = HOS 14:1 -HOS 14:1-9 = HOS 14:2-10 -JOL 2:28-32 = JOL 3:1-5 -JOL 3:1-21 = JOL 4:1-21 -JON 1:17 = JON 2:1 -JON 2:1-10 = JON 2:2-11 -MIC 5:1 = MIC 4:14 -MIC 5:2-15 = MIC 5:1-14 -NAM 1:15 = NAM 2:1 -NAM 2:1-13 = NAM 2:2-14 -ZEC 1:18-21 = ZEC 2:1-4 -ZEC 2:1-13 = ZEC 2:5-17 -MAL 4:1-6 = MAL 3:19-24 -ACT 19:40 = ACT 19:40 -ACT 19:41 = ACT 19:40 -ESG 1:1 = ESG 1:1a -ESG 1:2 = ESG 1:1b -ESG 1:3 = ESG 1:1c -ESG 1:4 = ESG 1:1d -ESG 1:5 = ESG 1:1e -ESG 1:6 = ESG 1:1f -ESG 1:7 = ESG 1:1g -ESG 1:8 = ESG 1:1h -ESG 1:9 = ESG 1:1i -ESG 1:10 = ESG 1:1k -ESG 1:11 = ESG 1:1l -ESG 1:12 = ESG 1:1m -ESG 1:13 = ESG 1:1n -ESG 1:14 = ESG 1:1o -ESG 1:15 = ESG 1:1p -ESG 1:16 = ESG 1:1q -ESG 1:17 = ESG 1:1r -ESG 1:18-39 = ESG 1:1-22 -ESG 3:14 = ESG 3:13a -ESG 3:15 = ESG 3:13b -ESG 3:16 = ESG 3:13c -ESG 3:17 = ESG 3:13d -ESG 3:18 = ESG 3:13e -ESG 3:19 = ESG 3:13f -ESG 3:20-22 = ESG 3:13-15 -ESG 4:18 = ESG 4:17a -ESG 4:19 = ESG 4:17b -ESG 4:20 = ESG 4:17c -ESG 4:21 = ESG 4:17c -ESG 4:22 = ESG 4:17d -ESG 4:23 = ESG 4:17d -ESG 4:24 = ESG 4:17e -ESG 4:25 = ESG 4:17f -ESG 4:26 = ESG 4:17g -ESG 4:27 = ESG 4:17h -ESG 4:28 = ESG 4:17i -ESG 4:29 = ESG 4:17k -ESG 4:30 = ESG 4:17k -ESG 4:31 = ESG 4:17k -ESG 4:32 = ESG 4:17l -ESG 4:33 = ESG 4:17m -ESG 4:34 = ESG 4:17n -ESG 4:35 = ESG 4:17n -ESG 4:36 = ESG 4:17o -ESG 4:37 = ESG 4:17o -ESG 4:38 = ESG 4:17p -ESG 4:39 = ESG 4:17q -ESG 4:40 = ESG 4:17r -ESG 4:41 = ESG 4:17s -ESG 4:42 = ESG 4:17t -ESG 4:43 = ESG 4:17u -ESG 4:44 = ESG 4:17w -ESG 4:45 = ESG 4:17x -ESG 4:46 = ESG 4:17y -ESG 4:47 = ESG 4:17z -ESG 5:2 = ESG 5:1a -ESG 5:3 = ESG 5:1a -ESG 5:4 = ESG 5:1a -ESG 5:5 = ESG 5:1b -ESG 5:6 = ESG 5:1c -ESG 5:7 = ESG 5:1d -ESG 5:8 = ESG 5:1e -ESG 5:9 = ESG 5:1f -ESG 5:10-11 = ESG 5:1-2 -ESG 5:12 = ESG 5:2 -ESG 5:13 = ESG 5:2a -ESG 5:14 = ESG 5:2a -ESG 5:15 = ESG 5:2b -ESG 5:16-28 = ESG 5:2-14 -ESG 8:13 = ESG 8:12a -ESG 8:14 = ESG 8:12b -ESG 8:15 = ESG 8:12c -ESG 8:16 = ESG 8:12d -ESG 8:17 = ESG 8:12e -ESG 8:18 = ESG 8:12f -ESG 8:19 = ESG 8:12g -ESG 8:20 = ESG 8:12h -ESG 8:21 = ESG 8:12i -ESG 8:22 = ESG 8:12k -ESG 8:23 = ESG 8:12l -ESG 8:24 = ESG 8:12m -ESG 8:25 = ESG 8:12n -ESG 8:26 = ESG 8:12o -ESG 8:27 = ESG 8:12p -ESG 8:28 = ESG 8:12q -ESG 8:29 = ESG 8:12r -ESG 8:30 = ESG 8:12s -ESG 8:31 = ESG 8:12t -ESG 8:32 = ESG 8:12u -ESG 8:33 = ESG 8:12x -ESG 8:34 = ESG 8:12y -ESG 8:35 = ESG 8:12y -ESG 8:36-41 = ESG 8:12-17 -ESG 10:4 = ESG 10:3a -ESG 10:5 = ESG 10:3b -ESG 10:6 = ESG 10:3c -ESG 10:7 = ESG 10:3d -ESG 10:8 = ESG 10:3e -ESG 10:9 = ESG 10:3f -ESG 10:10 = ESG 10:3g -ESG 10:11 = ESG 10:3h -ESG 10:12 = ESG 10:3i -ESG 10:13 = ESG 10:3k -ESG 10:14 = ESG 10:3l -BAR 6:1-73 = LJE 1:1-73 -S3Y 1:1-29 = DAG 3:24-52 -S3Y 1:30-31 = DAG 3:52-53 -S3Y 1:32 = DAG 3:55 -S3Y 1:33 = DAG 3:54 -S3Y 1:34-35 = DAG 3:56-57 -S3Y 1:36 = DAG 3:59 -S3Y 1:37 = DAG 3:58 -S3Y 1:38-68 = DAG 3:60-90 -DAG 13:1-63 = SUS 1:1-63 -DAG 14:1-42 = BEL 1:1-42 -# -# Excluded verses -#! -ROM 16:26 -#! -ROM 16:27 -#! -3JN 1:15 -#! -S3Y 1:49 -#! -ESG 4:6 -#! -ESG 9:5 -#! -ESG 9:30 -# -# Verse segment information diff --git a/samples/data/WEB-PT/631JNengWEB14.SFM b/samples/data/WEB-PT/631JNengWEB14.SFM deleted file mode 100644 index 2b4eb194d..000000000 --- a/samples/data/WEB-PT/631JNengWEB14.SFM +++ /dev/null @@ -1,199 +0,0 @@ -\id 1JN 62-1JN-web.sfm World English Bible (WEB) -\ide UTF-8 -\h 1 John -\toc1 John’s First Letter -\toc2 1 John -\toc3 1 John -\mt1 John’s First Letter -\c 1 -\p -\v 1 That which was from the beginning, that which we have heard, that which we have seen with our eyes, that which we saw, and our hands touched, concerning the Word of life -\v 2 (and the life was revealed, and we have seen, and testify, and declare to you the life, the eternal life, which was with the Father, and was revealed to us); -\v 3 that which we have seen and heard we declare to you, that you also may have fellowship with us. Yes, and our fellowship is with the Father and with his Son, Jesus Christ.\f + \fr 1:3 \ft “Christ” means “Anointed One”.\f* -\v 4 And we write these things to you, that our joy may be fulfilled. -\p -\v 5 This is the message which we have heard from him and announce to you, that God is light, and in him is no darkness at all. -\v 6 If we say that we have fellowship with him and walk in the darkness, we lie and don’t tell the truth. -\v 7 But if we walk in the light as he is in the light, we have fellowship with one another, and the blood of Jesus Christ his Son, cleanses us from all sin. -\v 8 If we say that we have no sin, we deceive ourselves, and the truth is not in us. -\v 9 If we confess our sins, he is faithful and righteous to forgive us the sins and to cleanse us from all unrighteousness. -\v 10 If we say that we haven’t sinned, we make him a liar, and his word is not in us. -\c 2 - -\p -\v 1 My little children, I write these things to you so that you may not sin. If anyone sins, we have a -Counselor\f + \fr 2:1 \ft Greek παρακλητον: Counselor, Helper, Intercessor, Advocate, and -Comforter.\f* with the Father, Jesus Christ, the righteous. -\v 2 And he is the atoning sacrifice\f + \fr 2:2 \ft “atoning sacrifice” is from the Greek “ιλασμος”, an appeasing, -propitiating, or the means of appeasement or propitiation—the sacrifice that turns away God’s wrath because of our sin. -\f* for our sins, and not for ours only, but also for the whole world. -\v 3 This is how we know that we know him: if we keep his commandments. -\v 4 One who says, “I know him,” and doesn’t keep his commandments, is a liar, and the truth isn’t in him. -\v 5 But God’s love has most certainly been perfected in whoever keeps his word. This is how we know that we are in him: -\v 6 he who says he remains in him ought himself also to walk just like he walked. -\p -\v 7 Brothers, I write no new commandment to you, but an old commandment which you had from the beginning. The old commandment -is the word which you heard from the beginning. -\v 8 Again, I write a new commandment to you, which is true in him and in you, because the darkness is passing away and the -true light already shines. -\v 9 He who says he is in the light and hates his brother is in the darkness even until now. -\v 10 He who loves his brother remains in the light, and there is no occasion for stumbling in him. -\v 11 But he who hates his brother is in the darkness, and walks in the darkness, and doesn’t know where he is going, because -the darkness has blinded his eyes. -\p -\v 12 I write to you, little children, because your sins are forgiven you for his name’s sake. -\p -\v 13 I write to you, fathers, because you know him who is from the beginning. -\p I write to you, young men, because you have overcome the evil one. -\p I write to you, little children, because you know the Father. -\p -\v 14 I have written to you, fathers, because you know him who is from the beginning. -\p I have written to you, young men, because you are strong, and the word of God remains in you, and you have overcome the evil -one. -\p -\v 15 Don’t love the world or the things that are in the world. If anyone loves the world, the Father’s love isn’t in him. -\v 16 For all that is in the world—the lust of the flesh, the lust of the eyes, and the pride of life—isn’t the Father’s, but -is the world’s. -\v 17 The world is passing away with its lusts, but he who does God’s will remains forever. -\p -\v 18 Little children, these are the end times, and as you heard that the Antichrist is coming, even now many antichrists have -arisen. By this we know that it is the final hour. -\v 19 They went out from us, but they didn’t belong to us; for if they had belonged to us, they would have continued with us. -But they left, that they might be revealed that none of them belong to us. -\v 20 You have an anointing from the Holy One, and you all have knowledge.\f + \fr 2:20 \ft Or, “know what is true”, or, “know all things”\f* -\v 21 I have not written to you because you don’t know the truth, but because you know it, and because no lie is of the truth. -\v 22 Who is the liar but he who denies that Jesus is the Christ? This is the Antichrist, he who denies the Father and the Son. -\v 23 Whoever denies the Son doesn’t have the Father. He who confesses the Son has the Father also. -\p -\v 24 Therefore, as for you, let that remain in you which you heard from the beginning. If that which you heard from the -beginning remains in you, you also will remain in the Son, and in the Father. -\v 25 This is the promise which he promised us, the eternal life. -\p -\v 26 These things I have written to you concerning those who would lead you astray. -\v 27 As for you, the anointing which you received from him remains in you, and you don’t need for anyone to teach you. But as -his anointing teaches you concerning all things, and is true, and is no lie, and even as it taught you, you will remain in him. -\p -\v 28 Now, little children, remain in him, that when he appears, we may have boldness and not be ashamed before him at his -coming. -\v 29 If you know that he is righteous, you know that everyone who practices righteousness has been born of him. -\c 3 - -\p -\v 1 See how great a love the Father has given to us, that we should be called children of God! For this cause the world -doesn’t know us, because it didn’t know him. -\v 2 Beloved, now we are children of God. It is not yet revealed what we will be; but we know that when he is revealed, we will -be like him, for we will see him just as he is. -\v 3 Everyone who has this hope set on him purifies himself, even as he is pure. -\p -\v 4 Everyone who sins also commits lawlessness. Sin is lawlessness. -\v 5 You know that he was revealed to take away our sins, and no sin is in him. -\v 6 Whoever remains in him doesn’t sin. Whoever sins hasn’t seen him and doesn’t know him. -\p -\v 7 Little children, let no one lead you astray. He who does righteousness is righteous, even as he is righteous. -\v 8 He who sins is of the devil, for the devil has been sinning from the beginning. To this end the Son of God was revealed: -that he might destroy the works of the devil. -\v 9 Whoever is born of God doesn’t commit sin, because his seed remains in him, and he can’t sin, because he is born of God. -\v 10 In this the children of God are revealed, and the children of the devil. Whoever doesn’t do righteousness is not of God, -neither is he who doesn’t love his brother. -\v 11 For this is the message which you heard from the beginning, that we should love one another— -\v 12 unlike Cain, who was of the evil one and killed his brother. Why did he kill him? Because his deeds were evil, and his -brother’s righteous. -\p -\v 13 Don’t be surprised, my brothers, if the world hates you. -\v 14 We know that we have passed out of death into life, because we love the brothers. He who doesn’t love his brother remains -in death. -\v 15 Whoever hates his brother is a murderer, and you know that no murderer has eternal life remaining in him. -\p -\v 16 By this we know love, because he laid down his life for us. And we ought to lay down our lives for the brothers. -\v 17 But whoever has the world’s goods and sees his brother in need, then closes his heart of compassion against him, how does -God’s love remain in him? -\p -\v 18 My little children, let’s not love in word only, or with the tongue only, but in deed and truth. -\v 19 And by this we know that we are of the truth and persuade our hearts before him, -\v 20 because if our heart condemns us, God is greater than our heart, and knows all things. -\v 21 Beloved, if our hearts don’t condemn us, we have boldness toward God; -\v 22 so whatever we ask, we receive from him, because we keep his commandments and do the things that are pleasing in his -sight. -\v 23 This is his commandment, that we should believe in the name of his Son, Jesus Christ, and love one another, even as he -commanded. -\v 24 He who keeps his commandments remains in him, and he in him. By this we know that he remains in us, by the Spirit which -he gave us. -\c 4 - - - -\p -\v 1 Beloved, don’t believe every spirit, but test the spirits, whether they are of God, because many false prophets have gone -out into the world. -\v 2 By this you know the Spirit of God: every spirit who confesses that Jesus Christ has come in the flesh is of God, -\v 3 and every spirit who doesn’t confess that Jesus Christ has come in the flesh is not of God; and this is the spirit of the -Antichrist, of whom you have heard that it comes. Now it is in the world already. -\v 4 You are of God, little children, and have overcome them, because greater is he who is in you than he who is in the world. -\v 5 They are of the world. Therefore they speak of the world, and the world hears them. -\v 6 We are of God. He who knows God listens to us. He who is not of God doesn’t listen to us. By this we know the spirit of -truth, and the spirit of error. -\p -\v 7 Beloved, let’s love one another, for love is of God; and everyone who loves has been born of God and knows God. -\v 8 He who doesn’t love doesn’t know God, for God is love. -\v 9 By this God’s love was revealed in us, that God has sent his only born\f + \fr 4:9 \ft The phrase “only born” is from the Greek word “μονογενη”, which is sometimes translated “only begotten” or “one and only”.\f* Son into the world that we might live through -him. -\v 10 In this is love, not that we loved God, but that he loved us, and sent his Son as the atoning -sacrifice\f + \fr 4:10 \ft “atoning sacrifice” is from the Greek “ιλασμος”, an appeasing, propitiating, or the means of -appeasement or propitiation—the sacrifice that turns away God’s wrath because of our sin. -\f* for our sins. -\v 11 Beloved, if God loved us in this way, we also ought to love one another. -\v 12 No one has seen God at any time. If we love one another, God remains in us, and his love has been perfected in us. -\p -\v 13 By this we know that we remain in him and he in us, because he has given us of his Spirit. -\v 14 We have seen and testify that the Father has sent the Son as the Savior of the world. -\v 15 Whoever confesses that Jesus is the Son of God, God remains in him, and he in God. -\v 16 We know and have believed the love which God has for us. God is love, and he who remains in love remains in God, and God -remains in him. -\v 17 In this, love has been made perfect among us, that we may have boldness in the day of judgment, because as he is, even so -we are in this world. -\v 18 There is no fear in love; but perfect love casts out fear, because fear has punishment. He who fears is not made perfect -in love. -\v 19 We love him,\f + \fr 4:19\ft NU omits “him”.\f* because he first loved us. -\v 20 If a man says, “I love God,” and hates his brother, he is a liar; for he who doesn’t love his brother whom he has seen, -how can he love God whom he has not seen? -\v 21 This commandment we have from him, that he who loves God should also love his brother. -\c 5 - -\p -\v 1 Whoever believes that Jesus is the Christ has been born of God. Whoever loves the Father also loves the child who is born -of him. -\v 2 By this we know that we love the children of God, when we love God and keep his commandments. -\v 3 For this is loving God, that we keep his commandments. His commandments are not grievous. -\v 4 For whatever is born of God overcomes the world. This is the victory that has overcome the world: your faith. -\v 5 Who is he who overcomes the world, but he who believes that Jesus is the Son of God? -\p -\v 6 This is he who came by water and blood, Jesus Christ; not with the water only, but with the water and the blood. It is the -Spirit who testifies, because the Spirit is the truth. -\v 7 For there are three who testify:\f + \fr 5:7 \ft Only a few recent manuscripts add “in heaven: the Father, the Word, and -the Holy Spirit; and these three are one. And there are three that testify on -earth:”\f* -\v 8 the Spirit, the water, and the blood; and the three agree as one. -\v 9 If we receive the witness of men, the witness of God is greater; for this is God’s testimony which he has testified -concerning his Son. -\v 10 He who believes in the Son of God has the testimony in himself. He who doesn’t believe God has made him a liar, because -he has not believed in the testimony that God has given concerning his Son. -\v 11 The testimony is this: that God gave to us eternal life, and this life is in his Son. -\v 12 He who has the Son has the life. He who doesn’t have God’s Son doesn’t have the life. -\p -\v 13 These things I have written to you who believe in the name of the Son of God, that you may know that you have eternal -life, and that you may continue to believe in the name of the Son of God. -\p -\v 14 This is the boldness which we have toward him, that if we ask anything according to his will, he listens to us. -\v 15 And if we know that he listens to us, whatever we ask, we know that we have the petitions which we have asked of him. -\p -\v 16 If anyone sees his brother sinning a sin not leading to death, he shall ask, and God will give him life for those who sin -not leading to death. There is sin leading to death. I don’t say that he should make a request concerning this. -\v 17 All unrighteousness is sin, and there is sin not leading to death. -\p -\v 18 We know that whoever is born of God doesn’t sin, but he who was born of God keeps himself, and the evil one doesn’t touch -him. -\v 19 We know that we are of God, and the whole world lies in the power of the evil one. -\v 20 We know that the Son of God has come and has given us an understanding, that we know him who is true; and we are in him -who is true, in his Son Jesus Christ. This is the true God and eternal life. -\p -\v 21 Little children, keep yourselves from idols. diff --git a/samples/data/WEB-PT/642JNengWEB14.SFM b/samples/data/WEB-PT/642JNengWEB14.SFM deleted file mode 100644 index ab7eaa1e2..000000000 --- a/samples/data/WEB-PT/642JNengWEB14.SFM +++ /dev/null @@ -1,25 +0,0 @@ -\id 2JN 63-2JN-web.sfm World English Bible (WEB) -\ide UTF-8 -\h 2 John -\toc1 John’s Second Letter -\toc2 2 John -\toc3 2 John -\mt1 John’s Second Letter -\c 1 -\p -\v 1 The elder, to the chosen lady and her children, whom I love in truth, and not I only, but also all those who know the truth, -\v 2 for the truth’s sake, which remains in us, and it will be with us forever: -\v 3 Grace, mercy, and peace will be with us, from God the Father and from the Lord Jesus Christ,\f + \fr 1:3 \ft “Christ” means “Anointed One”.\f* the Son of the Father, in truth and love. -\p -\v 4 I rejoice greatly that I have found some of your children walking in truth, even as we have been commanded by the Father. -\v 5 Now I beg you, dear lady, not as though I wrote to you a new commandment, but that which we had from the beginning, that we love one another. -\v 6 This is love, that we should walk according to his commandments. This is the commandment, even as you heard from the beginning, that you should walk in it. -\p -\v 7 For many deceivers have gone out into the world, those who don’t confess that Jesus Christ came in the flesh. This is the deceiver and the Antichrist. -\v 8 Watch yourselves, that we don’t lose the things which we have accomplished, but that we receive a full reward. -\v 9 Whoever transgresses and doesn’t remain in the teaching of Christ doesn’t have God. He who remains in the teaching has both the Father and the Son. -\v 10 If anyone comes to you and doesn’t bring this teaching, don’t receive him into your house, and don’t welcome him, -\v 11 for he who welcomes him participates in his evil deeds. -\p -\v 12 Having many things to write to you, I don’t want to do so with paper and ink, but I hope to come to you and to speak face to face, that our joy may be made full. -\v 13 The children of your chosen sister greet you. Amen. diff --git a/samples/data/WEB-PT/653JNengWEB14.SFM b/samples/data/WEB-PT/653JNengWEB14.SFM deleted file mode 100644 index 55d4fdcc7..000000000 --- a/samples/data/WEB-PT/653JNengWEB14.SFM +++ /dev/null @@ -1,29 +0,0 @@ -\id 3JN 64-3JN-web.sfm World English Bible (WEB) -\ide UTF-8 -\h 3 John -\toc1 John’s Third Letter -\toc2 3 John -\toc3 3 John -\mt1 John’s Third Letter -\c 1 -\p -\v 1 The elder to Gaius the beloved, whom I love in truth. -\p -\v 2 Beloved, I pray that you may prosper in all things and be healthy, even as your soul prospers. -\v 3 For I rejoiced greatly when brothers came and testified about your truth, even as you walk in truth. -\v 4 I have no greater joy than this: to hear about my children walking in truth. -\p -\v 5 Beloved, you do a faithful work in whatever you accomplish for those who are brothers and strangers. -\v 6 They have testified about your love before the assembly. You will do well to send them forward on their journey in a way worthy of God, -\v 7 because for the sake of the Name they went out, taking nothing from the Gentiles. -\v 8 We therefore ought to receive such, that we may be fellow workers for the truth. -\p -\v 9 I wrote to the assembly, but Diotrephes, who loves to be first among them, doesn’t accept what we say. -\v 10 Therefore, if I come, I will call attention to his deeds which he does, unjustly accusing us with wicked words. Not content with this, he doesn’t receive the brothers himself, and those who would, he forbids and throws out of the assembly. -\p -\v 11 Beloved, don’t imitate that which is evil, but that which is good. He who does good is of God. He who does evil hasn’t seen God. -\v 12 Demetrius has the testimony of all, and of the truth itself; yes, we also testify, and you know that our testimony is true. -\p -\v 13 I had many things to write to you, but I am unwilling to write to you with ink and pen; -\v 14 but I hope to see you soon. Then we will speak face to face. -\p Peace be to you. The friends greet you. Greet the friends by name. diff --git a/samples/data/WEB-PT/BookNames.xml b/samples/data/WEB-PT/BookNames.xml deleted file mode 100644 index 3681a1fb2..000000000 --- a/samples/data/WEB-PT/BookNames.xml +++ /dev/null @@ -1,126 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/samples/data/WEB-PT/Settings.xml b/samples/data/WEB-PT/Settings.xml deleted file mode 100644 index 2731664f1..000000000 --- a/samples/data/WEB-PT/Settings.xml +++ /dev/null @@ -1,86 +0,0 @@ - - 4 - engWEB14 - 65001 - English - Gentium Basic - 14 - engWEB14.SFM - 41MAT - usfm.sty - 8.0.63.1 - World English Bible (American Edition) - <p>PUBLIC DOMAIN</p> - Major::BiblicalTerms.xml - T - No - No - /home/kahunapule/Paratext8Projects/engweb18 - 9879dbb7cfe39e4d5d6f7f9dbd0c6414691a036e - 0 - “ ” - ‘ ’ - “ ” - Yes - No - p p/q p/q1 - T - - - , - : - – - ; - ; - Abbreviation - ShortName - False - a A b B c C d D e E f F g G h H i I j J k K l L m M n N o O p P q Q r R s S t T u U v V w W x X y Y z Z - * ‌ ‍ υ – + κ ρ π τ ν ι ς σ η λ μ % α = δ ἰ ὺ " ο & / ה נֵּ הִ ם אֱ לֹ הִ֑ י [ ] — 7 9 8 ! ’ “ ; 6 , : ) ( 2 . ” 3 5 0 ? 1 ⌃ ‘ 4 ' _ ִ֑י ֵּה ִנ ֹה א ֱל ῆ ῥ - \ ½ × χ ῦ θ ἄ β צִ έ ת צִ֛ ᾳ ἐ Μ ά ί ω ε ° έ ά נֵּ - γ - > < - _— ]_ _[ '_ – _=_ %_ ‘ -_ ) _&_ _—_ / ⌃_ ” ? _+_ —_ !_ . — _& - ?_ ..._ , ... _( )_ ._ ,_ _“ ”_ ;_ ’_ _‘ :_ ’ * ‌ ‍ ___ × °_ “ - “_ _?_ _’_ .._ _(_ _] _;_ ; _-_ _:_ _“_ _" "_ _._ _,_ - (/) [/] {/} - No - ib iq iq1 iq2 iq3 q q1 q2 q3 q4 qc qr qs qa qac qm qm1 qm2 qm3 b - io io1 io2 io3 io4 - Standard:: - True - Yes - a b c d e f g h i j k l m n o p q r s t u v w x y z aa bb cc dd ee ff gg hh ii jj kk ll mm nn oo pp qq rr ss tt uu vv ww xx yy zz - No - Yes - “/“ ‘ ’ ” - Yes - Yes - NIV84 - (none) - Compare - Invalid pair: - Could not find: - en::: - 2014-09-24T04:59:02.500278 - False - False - False - MatchVerseExactly - - - - See | LXX - 111111111111111111111111111111111111111111111111111111111111111111111111000011111111000000000000000100000000100100000000000 - - - dtyynkiqGHSk8qEp7 - Public - 546a4d9f5117ad7230055b04 - English(eng), World English Bible, United States, Bible with Deuterocanon, Published, eBible.org, engWEB14, dbl: matches, tms - False - - F - F - F - - - - \ No newline at end of file diff --git a/samples/data/WEB-PT/custom.sty b/samples/data/WEB-PT/custom.sty deleted file mode 100644 index b84499be3..000000000 --- a/samples/data/WEB-PT/custom.sty +++ /dev/null @@ -1,353 +0,0 @@ -# Custom style file created by the upgrade to Paratext 8.0. It is based upon -# the original style files used by your project. -# These files have been removed, but a backup of the files has been saved to -# usfm.sty.bak and custom.sty.bak. - - -\Marker imte3 -\Name imte3 - Introduction - [Uncommon] Major Title at Introduction End Level 3 -\Description Introduction major title at introduction end, level 3 -\OccursUnder id -\Bold -\FontSize 14 -\Rank 7 -\SpaceAfter 2 -\Justification center -\StyleType paragraph -\TextType other -\TextProperties paragraph publishable vernacular level_3 - -\Marker v -\OccursUnder lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 s3 d sp - -\Marker va -\OccursUnder lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tc1 tc2 tc3 tc4 s3 d - -\Marker vp -\OccursUnder lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tc1 tc2 tc3 tc4 s3 - -\Marker po -\Name po - Paragraph - Letter Opening -\Description Letter opening -\OccursUnder c -\FontSize 12 -\Rank 4 -\SpaceAfter 4 -\SpaceBefore 4 -\FirstLineIndent 0.125 -\StyleType paragraph -\TextType versetext -\TextProperties paragraph publishable vernacular - -\Marker cls -\Description Letter Closing - -\Marker pmo -\OccursUnder m mi nb p pc ph phi pi pi1 pi2 pi3 pr po q q1 q2 q3 q4 qc qr s1 s2 s3 s4 - -\Marker pm -\OccursUnder m mi nb p pc ph phi pi pi1 pi2 pi3 pr po psi q q1 q2 q3 q4 qc qr b s1 s2 s3 s4 - -\Marker pmc -\OccursUnder m mi nb p pc ph phi pi pi1 pi2 pi3 pr po q q1 q2 q3 q4 qc qr b s1 s2 s3 s4 - -\Marker pmr -\OccursUnder m mi nb p pc ph phi pi pi1 pi2 pi3 pr po q q1 q2 q3 q4 qc qr b s1 s2 s3 s4 - -\Marker qs -\OccursUnder q q1 q2 q3 q4 qc qr qd NEST - -\Marker qd -\Name qd - Poetry - Hebrew Note -\Description A Hebrew musical performance annotation, similar in content to Hebrew descriptive title. -\OccursUnder c -\Italic -\FontSize 12 -\Rank 4 -\LeftMargin 0.250 -\StyleType paragraph -\TextType versetext -\TextProperties paragraph publishable vernacular poetic - -\Marker mte1 -\OccursUnder lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tc1 tc2 tc3 tc4 s3 d - -\Marker lh -\Name lh - List Header -\Description List header (introductory remark) -\OccursUnder c -\FontSize 12 -\Rank 4 -\FirstLineIndent 0.125 -\StyleType paragraph -\TextType versetext -\TextProperties paragraph publishable vernacular - -\Marker li -\Description A list item, level 1 (if single level) - -\Marker li1 -\Description A list item, level 1 (if multiple levels) - -\Marker li2 -\Description A list item, level 2 - -\Marker li3 -\Description A list item, level 3 - -\Marker li4 -\Description A list item, level 4 - -\Marker lf -\Name lf - List Footer -\Description List footer (concluding remark) -\OccursUnder c -\FontSize 12 -\Rank 4 -\StyleType paragraph -\TextType versetext -\TextProperties paragraph publishable vernacular - -\Marker lim -\Name lim - Embedded List Item - Level 1 -\Description An embedded list item, level 1 (if single level) -\OccursUnder c -\FontSize 12 -\Rank 4 -\LeftMargin 0.750 -\RightMargin 0.250 -\FirstLineIndent -0.375 -\StyleType paragraph -\TextType versetext -\TextProperties paragraph publishable vernacular level_1 - -\Marker lim1 -\Name lim1 - Embedded List Item - Level 1 -\Description An embedded list item, level 1 (if multiple levels) -\OccursUnder c -\FontSize 12 -\Rank 4 -\LeftMargin 0.750 -\RightMargin 0.250 -\FirstLineIndent -0.375 -\StyleType paragraph -\TextType versetext -\TextProperties paragraph publishable vernacular level_1 - -\Marker lim2 -\Name lim2 - Embedded List Item - Level 2 -\Description An embedded list item, level 2 -\OccursUnder c -\FontSize 12 -\Rank 4 -\LeftMargin 1.000 -\FirstLineIndent -0.375 -\StyleType paragraph -\TextType versetext -\TextProperties paragraph publishable vernacular level_2 - -\Marker lim3 -\Name lim3 - Embedded List Item - Level 3 -\Description An embedded list item, level 3 -\OccursUnder c -\FontSize 12 -\Rank 4 -\LeftMargin 1.250 -\FirstLineIndent -0.375 -\StyleType paragraph -\TextType versetext -\TextProperties paragraph publishable vernacular level_3 - -\Marker lim4 -\Name lim4 - Embedded List Item - Level 4 -\Description An embedded list item, level 4 -\OccursUnder c -\FontSize 12 -\Rank 4 -\LeftMargin 1.500 -\FirstLineIndent -0.375 -\StyleType paragraph -\TextType versetext -\TextProperties paragraph publishable vernacular level_4 - -\Marker f -\OccursUnder c cp lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 qs sp tc1 tc2 tc3 tc4 mt mt1 mt2 mt3 ms ms1 ms2 ms3 s s1 s2 s3 d ip - -\Marker fe -\OccursUnder c lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 sp tc1 tc2 tc3 tc4 ms ms1 ms2 ms3 s s1 s2 s3 d ip - -\Marker fm -\OccursUnder lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 sp tc1 tc2 tc3 tc4 ms ms1 ms2 s s1 s2 s3 d ip - -\Marker x -\OccursUnder lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 qs sp tc1 tc2 tc3 tc4 mt mt1 mt2 mt3 ms ms1 ms2 s s1 s2 s3 d - -\Marker xop -\Name xop - Cross Reference - Origin Reference Publishing Alternate -\Description Published cross reference origin reference (origin reference that should appear in the published text) -\Endmarker xop* -\OccursUnder x -\FontSize 12 -\StyleType character -\TextType notetext -\TextProperties publishable vernacular note - -\Marker xta -\Name xta - Cross Reference - Target References Added Text -\Description Cross reference target references added text -\Endmarker xta* -\OccursUnder x -\FontSize 12 -\StyleType character -\TextType notetext -\TextProperties publishable vernacular note - -\Marker rq -\OccursUnder lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 NEST - -\Marker qt -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe NEST - -\Marker nd -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe NEST - -\Marker tl -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 cls tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe NEST - -\Marker dc -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe NEST - -\Marker bk -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe NEST - -\Marker sig -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 cls tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe NEST - -\Marker pn -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 cls tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe NEST - -\Marker png -\Name png...png* - Special Text - Geographic Proper Name -\Description For a geographic proper name -\Endmarker png* -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 cls tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe NEST -\Underline -\FontSize 12 -\StyleType character -\TextType versetext -\TextProperties publishable vernacular - -\Marker addpn -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 cls tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe NEST - -\Marker wj -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe NEST - -\Marker k -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe NEST - -\Marker sls -\OccursUnder lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 sp tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe NEST - -\Marker ord -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe NEST - -\Marker add -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 cls tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe NEST - -\Marker it -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe x NEST - -\Marker bd -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe x NEST - -\Marker bdit -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe x NEST - -\Marker em -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe x NEST - -\Marker sc -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 io io1 io2 io3 io4 ms ms1 ms2 s s1 s2 s3 s4 cd sp d lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 f fe x NEST - -\Marker fig -\OccursUnder lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 sp tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 ms ms1 ms2 s s1 s2 s3 d ip - -\Marker pro -\OccursUnder ip im ipi imi ipq imq ipr iq iq1 iq2 iq3 lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 sp tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 ms ms1 ms2 s s1 s2 s3 d ip f fe NEST - -\Marker w -\OccursUnder f ip im lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po d q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 s s1 s2 s3 s4 NEST - -\Marker wh -\OccursUnder lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po d q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 s s1 s2 s3 s4 NEST - -\Marker wg -\OccursUnder lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po d q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 s s1 s2 s3 s4 NEST - -\Marker wa -\Name wa...wa* - Peripheral Ref - Aramaic Wordlist Entry -\Description An Aramaic Wordlist text item -\Endmarker wa* -\OccursUnder lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po d q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 s s1 s2 s3 s4 NEST -\FontSize 12 -\StyleType character -\TextType versetext -\TextProperties paragraph publishable vernacular - -\Marker ndx -\OccursUnder lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr pmo pm pmc pmr po q q1 q2 q3 q4 qc qr qd qm qm1 qm2 qm3 tr th1 th2 th3 th4 thr1 thr2 thr3 thr4 tc1 tc2 tc3 tc4 tcr1 tcr2 tcr3 tcr4 s s1 s2 s3 s4 NEST - -\Marker wr -\OccursUnder ms s lh li li1 li2 li3 li4 lf lim lim1 lim2 lim3 lim4 m mi nb p pc ph phi pi pi1 pi2 pi3 pr po q q1 q2 q3 q4 qc qr qd tc1 tc2 tc3 tc4 f fe NEST - -\Marker ul -\Name ul...ul* - Character - Underline -\Description A character style, for underlined text -\Endmarker ul* -\OccursUnder ip ipi im imi ili ili1 ili2 iex io1 io2 io3 io4 cd li li1 li2 li3 li4 m mi nb p pc ph phi pi pr pmo pm pmc pmr q q1 q2 q3 q4 qc qr qm qm1 qm2 qm3 sp tc1 tc2 tc3 tc4 ms ms1 ms2 s s1 s2 s3 spd d ip f fe NEST -\Underline -\FontSize 12 -\StyleType character -\TextProperties publishable vernacular - -\Marker zw -\Name zw...zw* - Special Text - Strong's Number Start -\Endmarker zw* -\FontSize 7 -\StyleType character -\TextType versetext -\TextProperties publishable vernacular - -\Marker zx -\Name zx...zx* - Special Text - Strong's Number End -\Endmarker zx* -\FontSize 7 -\StyleType character -\TextType versetext -\TextProperties publishable vernacular - -\Marker toc4 -\Name toc4 - File - Alternate Book Name or Abbreviation -\Description Alternate Book Name or Abbreviation -\OccursUnder h h1 h2 h3 -\Italic -\Color 128 -\FontSize 12 -\Rank 1 -\StyleType paragraph -\TextType other -\TextProperties paragraph publishable vernacular - -\Marker ztoc4 -\Name ztoc4 - File - Alternate Book Name or Abbreviation -\Description Alternate Book Name or Abbreviation -\OccursUnder h h1 h2 h3 -\Italic -\Color 128 -\FontSize 12 -\Rank 1 -\StyleType paragraph -\TextType other -\TextProperties paragraph publishable vernacular - diff --git a/samples/data/WEB-PT/custom.vrs b/samples/data/WEB-PT/custom.vrs deleted file mode 100644 index 3053290dc..000000000 --- a/samples/data/WEB-PT/custom.vrs +++ /dev/null @@ -1,30 +0,0 @@ -# custom.vrs -# prepared by Michael Johnson for the World English Bible project - -# amend total quantity of verses in chapter -ROM 14:26 -REV 12:17 -TOB 5:22 -TOB 10:12 -SIR 23:28 -ESG 1:22 -ESG 3:15 -ESG 5:14 -ESG 8:17 -ESG 10:14 -SIR 33:33 -SIR 41:24 -BAR 1:22 -4MA 7:25 -4MA 12:20 -# deliberately missing verses --ROM 16:26 --ROM 16:27 --3JN 1:15 --S3Y 1:49 --ESG 4:6 --ESG 9:5 --ESG 9:30 - - - diff --git a/samples/data/WEB-PT/en.ldml b/samples/data/WEB-PT/en.ldml deleted file mode 100644 index 365da17e0..000000000 --- a/samples/data/WEB-PT/en.ldml +++ /dev/null @@ -1 +0,0 @@ -[][!%-/\:;=?\[\]\u00B0\u00D7\u200C\u200D\u2013\u2014\u2018\u2019\u201C\u201D\u2303{...}][][][][a b c d e f g h i j k l m n o p q r s t u v w x y z {aa} {bb} {cc} {dd} {ee} {ff} {gg} {hh} {ii} {jj} {kk} {ll} {mm} {nn} {oo} {pp} {qq} {rr} {ss} {tt} {uu} {vv} {ww} {xx} {yy} {zz}][][]“”‘’left-to-rightstandard \ No newline at end of file diff --git a/samples/data/WEB-PT/frtbak_old.sty b/samples/data/WEB-PT/frtbak_old.sty deleted file mode 100644 index 8d20c474a..000000000 --- a/samples/data/WEB-PT/frtbak_old.sty +++ /dev/null @@ -1,2278 +0,0 @@ -# Default frtbak.sty created from usfm.sty - - -\Marker id -\Name id - File - Identification -\Description File identification information (BOOKID, FILENAME, EDITOR, MODIFICATION DATE) -\StyleType Paragraph -\TextType Other -\TextProperties paragraph nonpublishable nonvernacular book -\FontSize 12 - -\Marker ide -\Name ide - File - Encoding -\Description File encoding information -\StyleType Paragraph -\TextType Other -\TextProperties paragraph nonpublishable nonvernacular -\FontSize 12 - -# Headers - -\Marker h -\Name h - File - Header -\Description Running header text for a book (basic) -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 - -\Marker h1 -\Name h1 - File - Header -\Description Running header text -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 - -\Marker h2 -\Name h2 - File - Left Header -\Description Running header text, left side of page -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 - -\Marker h3 -\Name h3 - File - Right Header -\Description Running header text, right side of page -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 - -\Marker toc1 -\Name toc1 - File - Long Table of Contents Text -\Description Long table of contents text -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\Italic -\Bold -\color 16384 - -\Marker toc2 -\Name toc2 - File - Short Table of Contents Text -\Description Short table of contents text -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\Italic -\color 16384 - -\Marker toc3 -\Name toc3 - File - Book Abbreviation -\Description Book Abbreviation -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\Bold -\Italic -\color 128 - -# Remarks and Comments - -\Marker rem -\Name rem - File - Remark -\Description Comments and remarks -\TextType Other -\TextProperties paragraph nonpublishable nonvernacular -\StyleType Paragraph -\FontSize 12 -\Color 16711680 - -\Marker sts -\Name rem - File - Status -\Description Status of this file -\TextType Other -\TextProperties paragraph nonpublishable nonvernacular -\StyleType Paragraph -\FontSize 12 -\Color 16711680 - -\Marker restore -\Name restore - File - Restore Information -\Description Project restore information -\TextType Other -\TextProperties paragraph nonpublishable nonvernacular -\StyleType Paragraph -\FontSize 12 -\Color 16711680 - -# Introduction - -\Marker imt -\Name imt - Introduction - Major Title Level 1 -\Description Introduction major title, level 1 (if single level) (basic) -\TextProperties paragraph publishable vernacular level_1 -\TextType Other -\StyleType Paragraph -\FontSize 14 -\Bold -\Justification Center -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker imt1 -\Name imt1 - Introduction - Major Title Level 1 -\Description Introduction major title, level 1 (if multiple levels) -\TextProperties paragraph publishable vernacular level_1 -\TextType Other -\StyleType Paragraph -\FontSize 14 -\Bold -\Justification Center -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker imt2 -\Name imt2 - Introduction - Major Title Level 2 -\Description Introduction major title, level 2 -\TextProperties paragraph publishable vernacular level_2 -\TextType other -\StyleType Paragraph -\FontSize 13 -\Italic -\Justification Center -\SpaceBefore 6 -\SpaceAfter 3 - -\Marker imt3 -\Name imt3 - Introduction - Major Title Level 3 -\Description Introduction major title, level 3 -\TextProperties paragraph publishable vernacular level_3 -\TextType Other -\StyleType Paragraph -\FontSize 12 -\Bold -\Justification Center -\SpaceBefore 2 -\SpaceAfter 2 - -\Marker imt4 -\Name imt4 - Introduction - Major Title Level 4 -\Description Introduction major title, level 4 (usually within parenthesis) -\TextProperties paragraph publishable vernacular level_4 -\TextType Other -\StyleType Paragraph -\FontSize 12 -\Italic -\Justification Center -\SpaceBefore 2 -\SpaceAfter 2 - -\Marker imte -\Name imte - Introduction - [Uncommon] Major Title at Introduction End Level 1 -\Description Introduction major title at introduction end, level 1 (if single level) -\TextProperties paragraph publishable vernacular level_1 -\TextType Other -\StyleType Paragraph -\FontSize 20 -\Bold -\Justification Center -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker imte1 -\Name imte1 - Introduction - [Uncommon] Major Title at Introduction End Level 1 -\Description Introduction major title at introduction end, level 1 (if multiple levels) -\TextProperties paragraph publishable vernacular level_1 -\TextType Other -\StyleType Paragraph -\FontSize 20 -\Bold -\Justification Center -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker imte2 -\Name imte2 - Introduction - [Uncommon] Major Title at Introduction End Level 2 -\Description Introduction major title at introduction end, level 2 -\TextProperties paragraph publishable vernacular level_2 -\TextType Other -\StyleType Paragraph -\FontSize 16 -\Italic -\Justification Center -\SpaceAfter 2 - -\Marker is -\Name is - Introduction - Section Heading Level 1 -\Description Introduction section heading, level 1 (if single level) (basic) -\TextType Other -\TextProperties paragraph publishable vernacular level_1 -\StyleType Paragraph -\FontSize 14 -\Bold -\Justification Center -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker is1 -\Name is1 - Introduction - Section Heading Level 1 -\Description Introduction section heading, level 1 (if multiple levels) -\TextType Other -\TextProperties paragraph publishable vernacular level_1 -\StyleType Paragraph -\FontSize 14 -\Bold -\Justification Center -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker is2 -\Name is2 - Introduction - Section Heading Level 2 -\Description Introduction section heading, level 2 -\TextType Other -\TextProperties paragraph publishable vernacular level_2 -\StyleType Paragraph -\FontSize 12 -\Bold -\Justification Center -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker iot -\Name iot - Introduction - Outline Title -\Description Introduction outline title (basic) -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\Bold -\Justification Center -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker io -\Name io - Introduction - Outline Level 1 -\Description Introduction outline text, level 1 (if single level) -\TextType Other -\TextProperties paragraph publishable vernacular level_1 -\StyleType paragraph -\FontSize 12 -\LeftMargin .5 - -\Marker io1 -\Name io1 - Introduction - Outline Level 1 -\Description Introduction outline text, level 1 (if multiple levels) (basic) -\TextType Other -\TextProperties paragraph publishable vernacular level_1 -\StyleType paragraph -\FontSize 12 -\LeftMargin .5 - -\Marker io2 -\Name io2 - Introduction - Outline Level 2 -\Description Introduction outline text, level 2 -\TextType Other -\TextProperties paragraph publishable vernacular level_2 -\StyleType paragraph -\FontSize 12 -\LeftMargin .75 - -\Marker io3 -\Name io3 - Introduction - Outline Level 3 -\Description Introduction outline text, level 3 -\TextType Other -\TextProperties paragraph publishable vernacular level_3 -\StyleType paragraph -\FontSize 12 -\LeftMargin 1 - -\Marker io4 -\Name io4 - Introduction - Outline Level 4 -\Description Introduction outline text, level 4 -\TextType Other -\TextProperties paragraph publishable vernacular level_4 -\StyleType paragraph -\FontSize 12 -\LeftMargin 1.25 - -\Marker ior -\Endmarker ior* -\Name ior...ior* - Introduction - Outline References Range -\Description Introduction references range for outline entry; for marking references separately -\TextType Other -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 - -\Marker ip -\Name ip - Introduction - Paragraph -\Description Introduction prose paragraph (basic) -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\FirstLineIndent .125 # 1/8 inch first line indent - -\Marker im -\Name im - Introduction - Paragraph - no first line indent -\Description Introduction prose paragraph, with no first line indent (may occur after poetry) -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 - -\Marker ipi -\Name ipi - Introduction - Indented Para - first line indent -\Description Introduction prose paragraph, indented, with first line indent -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\FirstLineIndent .125 # 1/8 inch first line indent -\LeftMargin .25 -\RightMargin .25 - -\Marker imi -\Name imi - Introduction - Indented Para - no first line indent -\Description Introduction prose paragraph text, indented, with no first line indent -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\LeftMargin .25 -\RightMargin .25 - -\Marker ili -\Name ili - Introduction - List Entry - Level 1 -\Description A list entry, level 1 (if single level) -\TextType Other -\TextProperties paragraph publishable vernacular level_1 -\StyleType paragraph -\FontSize 12 -\LeftMargin .5 -\FirstLineIndent -.375 # 1/8 inch indent, 1/2 inch wrap - -\Marker ili1 -\Name ili1 - Introduction - List Entry - Level 1 -\Description A list entry, level 1 (if multiple levels) -\TextType Other -\TextProperties paragraph publishable vernacular level_1 -\StyleType paragraph -\FontSize 12 -\LeftMargin .5 -\FirstLineIndent -.375 # 1/8 inch indent, 1/2 inch wrap - -\Marker ili2 -\Name ili2 - Introduction - List Entry - Level 2 -\Description A list entry, level 2 -\TextType Other -\TextProperties paragraph publishable vernacular level_2 -\StyleType paragraph -\FontSize 12 -\LeftMargin .75 -\FirstLineIndent -.375 - -\Marker ipq -\Name ipq - Introduction - Paragraph - quote from text -\Description Introduction prose paragraph, quote from the body text -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\LeftMargin .25 -\RightMargin .25 -\FirstLineIndent .125 # 1/8 inch first line indent -\Italic - -\Marker imq -\Name imq - Introduction - Paragraph - quote from text - no first line indent -\Description Introduction prose paragraph, quote from the body text, with no first line indent -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\LeftMargin .25 -\RightMargin .25 -\Italic - -\Marker ipr -\Name ipr - Introduction - Paragraph - right aligned -\Description Introduction prose paragraph, right aligned -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\LeftMargin .25 -\RightMargin .25 -\Justification Right -\Italic - -\Marker ib -\Name ib - Introduction - Blank Line -\Description Introduction blank line -\TextProperties paragraph publishable vernacular poetic -\TextType Other -\FontSize 10 -\StyleType paragraph - -\Marker iq -\Name iq - Introduction - Poetry Level 1 -\Description Introduction poetry text, level 1 (if single level) -\TextType Other -\TextProperties paragraph publishable vernacular poetic level_1 -\StyleType Paragraph -\FontSize 12 -\LeftMargin 1 -\FirstLineIndent -.75 # 1/4 inch indent, 1 inch wrap -\Italic - -\Marker iq1 -\Name iq1 - Introduction - Poetry Level 1 -\Description Introduction poetry text, level 1 (if multiple levels) -\TextType Other -\TextProperties paragraph publishable vernacular poetic level_1 -\StyleType Paragraph -\FontSize 12 -\LeftMargin 1 -\FirstLineIndent -.75 # 1/4 inch indent, 1 inch wrap -\Italic - -\Marker iq2 -\Name iq2 - Introduction - Poetry Level 2 -\Description Introduction poetry text, level 2 -\TextType Other -\TextProperties paragraph publishable vernacular poetic level_2 -\StyleType Paragraph -\FontSize 12 -\LeftMargin 1 -\FirstLineIndent -.5 # 1/2 inch indent, 1 inch wrap -\Italic - -\Marker iq3 -\Name iq3 - Introduction - Poetry Level 3 -\Description Introduction poetry text, level 3 -\TextType Other -\TextProperties paragraph publishable vernacular poetic level_3 -\StyleType Paragraph -\FontSize 12 -\LeftMargin 1 -\FirstLineIndent -.25 # 3/4 inch indent, 1 inch wrap -\Italic - -\Marker iex -\Name iex - Introduction - Explanatory or Bridge Text -\Description Introduction explanatory or bridge text (e.g. explanation of missing book in Short Old Testament) -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\FirstLineIndent .125 # 1/8 inch first line indent -\SpaceBefore 4 -\SpaceAfter 4 - -\Marker iqt -\Endmarker iqt* -\Name iqt...iqt* - Special Text - Quoted Scripture Text in Introduction -\Description For quoted scripture text appearing in the introduction -\TextType Other -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic - -\Marker ie -\Name ie - Introduction - End Marker -\Description Introduction ending marker -\TextProperties paragraph publishable vernacular -\TextType Other -\FontSize 10 -\StyleType paragraph - -# Chapters and Verses - -\Marker c -\Name c - Chapter Number -\Description Chapter number (necessary for normal Paratext operation) -\TextType ChapterNumber -\TextProperties chapter -\StyleType Paragraph -\Bold -\FontSize 18 -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker ca -\Endmarker ca* -\Name ca...ca* - Chapter Number - Alternate -\Description Second (alternate) chapter number (for coding dual versification; useful for places where different traditions of chapter breaks need to be supported in the same translation) -\TextType Other -\StyleType Character -\Italic -\FontSize 16 -\Color 2263842 - -\Marker cp -\Name cp - Chapter Number - Publishing Alternate -\Description Published chapter number (this is a chapter marking that would be used in the published text) -\TextType Other -\TextProperties paragraph -\StyleType Paragraph -\Bold -\FontSize 18 -\SpaceBefore 8 -\SpaceAfter 4 -\Color 16711680 - -\Marker cl -\Name cl - Chapter - Publishing Label -\Description Chapter label used for translations that add a word such as "Chapter" before chapter numbers (e.g. Psalms). The subsequent text is the chapter label. -\TextType Other -\TextProperties paragraph -\StyleType Paragraph -\Bold -\FontSize 18 -\SpaceBefore 8 -\SpaceAfter 4 -\Justification Center - -\Marker cd -\Name cd - Chapter - Description -\Description Chapter Description (Publishing option D, e.g. in Russian Bibles) -\TextType Other -\TextProperties paragraph -\StyleType Paragraph -\FontSize 11 -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker v -\Name v - Verse Number -\Description A verse number (Necessary for normal paratext operation) (basic) -\TextType VerseNumber -\TextProperties verse -\StyleType Character -\FontSize 12 -\Superscript - -\Marker va -\Endmarker va* -\Name va...va* - Verse Number - Alternate -\Description Second (alternate) verse number (for coding dual numeration in Psalms; see also NRSV Exo 22.1-4) -\TextType Other -\StyleType Character -\FontSize 12 -\Superscript -\Color 2263842 - -\Marker vp -\Endmarker vp* -\Name vp...vp* - Verse Number - Publishing Alternate -\Description Published verse marker - this is a verse marking that would be used in the published text -\TextType Other -\StyleType Character -\FontSize 12 -\Superscript -\Color 16711680 - -# Paragraphs - -\Marker p -\Name p - Paragraph - Normal - First Line Indent -\Description Paragraph text, with first line indent (basic) -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\FirstLineIndent .125 # 1/8 inch first line indent - -\Marker m -\Name m - Paragraph - Margin - No First Line Indent -\Description Paragraph text, with no first line indent (may occur after poetry) (basic) -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 - -\Marker pmo -\Name pmo - Paragraph - Embedded Text Opening -\Description Embedded text opening -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\LeftMargin .25 -\RightMargin .25 - -\Marker pm -\Name pm - Paragraph - Embedded Text -\Description Embedded text paragraph -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\FirstLineIndent .125 # 1/8 inch first line indent -\LeftMargin .25 -\RightMargin .25 - -\Marker pmc -\Name pmc - Paragraph - Embedded Text Closing -\Description Embedded text closing -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\LeftMargin .25 -\RightMargin .25 - -\Marker pmr -\Name pmr - Paragraph - Embedded Text Refrain -\Description Embedded text refrain (e.g. Then all the people shall say, "Amen!") -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\Justification Right -\LeftMargin .25 -\RightMargin .25 - -\Marker pi -\Name pi - Paragraph - Indented - Level 1 - First Line Indent -\Description Paragraph text, level 1 indent (if sinlge level), with first line indent; often used for discourse (basic) -\TextType VerseText -\TextProperties paragraph publishable vernacular level_1 -\StyleType Paragraph -\FontSize 12 -\FirstLineIndent .125 # 1/8 inch first line indent -\LeftMargin .25 -\RightMargin .25 - -\Marker pi1 -\Name pi1 - Paragraph - Indented - Level 1 - First Line Indent -\Description Paragraph text, level 1 indent (if multiple levels), with first line indent; often used for discourse -\TextType VerseText -\TextProperties paragraph publishable vernacular level_1 -\StyleType Paragraph -\FontSize 12 -\FirstLineIndent .125 # 1/8 inch first line indent -\LeftMargin .25 -\RightMargin .25 - -\Marker pi2 -\Name pi2 - Paragraph - Indented - Level 2 - First Line Indent -\Description Paragraph text, level 2 indent, with first line indent; often used for discourse -\TextType VerseText -\TextProperties paragraph publishable vernacular level_2 -\StyleType Paragraph -\FontSize 12 -\FirstLineIndent .125 # 1/8 inch first line indent -\LeftMargin .5 -\RightMargin .25 - -\Marker pi3 -\Name pi3 - Paragraph - Indented - Level 3 - First Line Indent -\Description Paragraph text, level 3 indent, with first line indent; often used for discourse -\TextType VerseText -\TextProperties paragraph publishable vernacular level_3 -\StyleType Paragraph -\FontSize 12 -\FirstLineIndent .125 # 1/8 inch first line indent -\LeftMargin .75 -\RightMargin .25 - -\Marker pc -\Name pc - Paragraph - Centered (for Inscription) -\Description Paragraph text, centered (for Inscription) -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\Justification Center -\FontSize 12 - -\Marker mi -\Name mi - Paragraph - Indented - No First Line Indent -\Description Paragraph text, indented, with no first line indent; often used for discourse -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\LeftMargin .25 -\RightMargin .25 - -\Marker nb -\Name nb - Paragraph - No Break with Previous Paragraph -\Description Paragraph text, with no break from previous paragraph text (at chapter boundary) (basic) -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 - -\Marker cls -\Name cls - Paragraph - Closure of an Epistle -\Description Closure of an Epistle -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\Justification Right - -# Poetry - -\Marker q -\Name q - Poetry - Indent Level 1 - Single Level Only -\Description Poetry text, level 1 indent (if single level) -\TextType VerseText -\TextProperties paragraph publishable vernacular poetic level_1 -\StyleType Paragraph -\FontSize 12 -\LeftMargin 1.25 -\FirstLineIndent -1 # 1/4 inch indent, 1 inch wrap - -\Marker q1 -\Name q1 - Poetry - Indent Level 1 -\Description Poetry text, level 1 indent (if multiple levels) (basic) -\TextType VerseText -\TextProperties paragraph publishable vernacular poetic level_1 -\StyleType Paragraph -\FontSize 12 -\LeftMargin 1.25 -\FirstLineIndent -1 # 1/4 inch indent, 1 inch wrap - -\Marker q2 -\Name q2 - Poetry - Indent Level 2 -\Description Poetry text, level 2 indent (basic) -\TextType VerseText -\TextProperties paragraph publishable vernacular poetic level_2 -\StyleType Paragraph -\FontSize 12 -\LeftMargin 1.25 -\FirstLineIndent -.75 # 1/2 inch indent, 1 inch wrap - -\Marker q3 -\Name q3 - Poetry - Indent Level 3 -\Description Poetry text, level 3 indent -\TextType VerseText -\TextProperties paragraph publishable vernacular poetic level_3 -\StyleType Paragraph -\FontSize 12 -\LeftMargin 1.25 -\FirstLineIndent -.5 # 3/4 inch indent, 1 inch wrap - -\Marker q4 -\Name q4 - Poetry - Indent Level 4 -\Description Poetry text, level 4 indent -\TextType VerseText -\TextProperties paragraph publishable vernacular poetic level_3 -\StyleType Paragraph -\FontSize 12 -\LeftMargin 1.25 -\FirstLineIndent -.25 # 3/4 inch indent, 1 inch wrap - -\Marker qc -\Name qc - Poetry - Centered -\Description Poetry text, centered -\TextType VerseText -\TextProperties paragraph publishable vernacular poetic -\StyleType Paragraph -\FontSize 12 -\Justification Center - -\Marker qr -\Name qr - Poetry - Right Aligned -\Description Poetry text, Right Aligned -\TextType VerseText -\TextProperties paragraph publishable vernacular poetic -\StyleType Paragraph -\FontSize 12 -\Justification Right - -\Marker qs -\Endmarker qs* -\Name qs...qs* - Poetry Text - Selah -\Description Poetry text, Selah -\TextType VerseText -\TextProperties publishable vernacular poetic -\StyleType Character -\FontSize 12 -\Italic - -\Marker qa -\Name qa - Poetry - Acrostic Heading/Marker -\Description Poetry text, Acrostic marker/heading -\TextType Other -\TextProperties paragraph publishable vernacular poetic -\StyleType Paragraph -\FontSize 12 -\Italic - -\Marker qac -\Endmarker qac* -\Name qac...qac* - Poetry Text - Acrostic Letter -\Description Poetry text, Acrostic markup of the first character of a line of acrostic poetry -\TextType Other -\TextProperties publishable vernacular poetic -\StyleType Character -\FontSize 12 -\Italic - -\Marker qm -\Name qm - Poetry - Embedded Text - Indent Level 1 - Single Level Only -\Description Poetry text, embedded, level 1 indent (if single level) -\TextType VerseText -\TextProperties paragraph publishable vernacular poetic -\StyleType Paragraph -\FontSize 12 -\LeftMargin 1 -\FirstLineIndent -.75 # 1/4 inch indent, 1 inch wrap - -\Marker qm1 -\Name qm1 - Poetry - Embedded Text - Indent Level 1 -\Description Poetry text, embedded, level 1 indent (if multiple levels) -\TextType VerseText -\TextProperties paragraph publishable vernacular poetic level_1 -\StyleType Paragraph -\FontSize 12 -\LeftMargin 1 -\FirstLineIndent -.75 # 1/4 inch indent, 1 inch wrap - -\Marker qm2 -\Name qm2 - Poetry - Embedded Text - Indent Level 2 -\Description Poetry text, embedded, level 2 indent -\TextType VerseText -\TextProperties paragraph publishable vernacular poetic level_2 -\StyleType Paragraph -\FontSize 12 -\LeftMargin 1 -\FirstLineIndent -.5 # 1/2 inch indent, 1 inch wrap - -\Marker qm3 -\Name qm3 - Poetry - Embedded Text - Indent Level 3 -\Description Poetry text, embedded, level 3 indent -\TextType VerseText -\TextProperties paragraph publishable vernacular poetic level_3 -\StyleType Paragraph -\FontSize 12 -\LeftMargin 1 -\FirstLineIndent -.25 # 3/4 inch indent, 1 inch wrap - -\Marker b -\Name b - Poetry - Stanza Break (Blank Line) -\Description Poetry text stanza break (e.g. stanza break) (basic) -\TextProperties paragraph publishable vernacular poetic -\TextType VerseText -\FontSize 10 -\StyleType paragraph - -# Titles & Headings - -\Marker mt -\Name mt - Title - Major Title Level 1 -\Description The main title of the book (if single level) -\TextProperties paragraph publishable vernacular level_1 -\TextType Title -\StyleType Paragraph -\FontSize 20 -\Bold -\Justification Center -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker mt1 -\Name mt1 - Title - Major Title Level 1 -\Description The main title of the book (if multiple levels) (basic) -\TextProperties paragraph publishable vernacular level_1 -\TextType Title -\StyleType Paragraph -\FontSize 20 -\Bold -\Justification Center -\SpaceBefore 2 -\SpaceAfter 4 - -\Marker mt2 -\Name mt2 - Title - Major Title Level 2 -\Description A secondary title usually occurring before the main title (basic) -\TextProperties paragraph publishable vernacular level_2 -\TextType Title -\StyleType Paragraph -\FontSize 16 -\Italic -\Justification Center -\SpaceAfter 2 - -\Marker mt3 -\Name mt3 - Title - Major Title Level 3 -\Description A secondary title occurring after the main title -\TextProperties paragraph publishable vernacular level_3 -\TextType Title -\StyleType Paragraph -\FontSize 16 -\Bold -\Justification Center -\SpaceBefore 2 -\SpaceAfter 2 - -\Marker mt4 -\Name mt4 - Title - Major Title level 4 -\Description A small secondary title sometimes occuring within parentheses -\TextProperties paragraph publishable vernacular level_4 -\TextType Title -\StyleType Paragraph -\FontSize 12 -\Justification Center -\SpaceBefore 2 -\SpaceAfter 2 - -\Marker mte -\Name mte - Title - [Uncommon] Major Title Ending Level 1 -\Description The main title of the book repeated at the end of the book, level 1 (if single level) -\TextProperties paragraph publishable vernacular level_1 -\TextType Title -\StyleType Paragraph -\FontSize 20 -\Bold -\Justification Center -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker mte1 -\Name mte1 - Title - [Uncommon] Major Title Ending Level 1 -\Description The main title of the book repeated at the end of the book, level 1 (if multiple levels) -\TextProperties paragraph publishable vernacular level_1 -\TextType Title -\StyleType Paragraph -\FontSize 20 -\Bold -\Justification Center -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker mte2 -\Name mte2 - Title - [Uncommon] Major Title Ending Level 2 -\Description A secondary title occurring before or after the 'ending' main title -\TextProperties paragraph publishable vernacular level_2 -\TextType Title -\StyleType Paragraph -\FontSize 16 -\Italic -\Justification Center -\SpaceAfter 2 - -\Marker ms -\Name ms - Heading - Major Section Level 1 -\Description A major section division heading, level 1 (if single level) (basic) -\TextType Section -\TextProperties paragraph publishable vernacular level_1 -\StyleType Paragraph -\FontSize 14 -\Bold -\Justification Center -\SpaceBefore 16 -\SpaceAfter 4 - -\Marker ms1 -\Name ms1 - Heading - Major Section Level 1 -\Description A major section division heading, level 1 (if multiple levels) -\TextType Section -\TextProperties paragraph publishable vernacular level_1 -\StyleType Paragraph -\FontSize 14 -\Bold -\Justification Center -\SpaceBefore 16 -\SpaceAfter 4 - -\Marker ms2 -\Name ms2 - Heading - Major Section Level 2 -\Description A major section division heading, level 2 -\TextType Section -\TextProperties paragraph publishable vernacular level_1 -\StyleType Paragraph -\FontSize 14 -\Bold -\Justification Center -\SpaceBefore 16 -\SpaceAfter 4 - -\Marker ms3 -\Name ms3 - Heading - Major Section Level 3 -\Description A major section division heading, level 3 -\TextType Section -\TextProperties paragraph publishable vernacular level_1 -\StyleType Paragraph -\FontSize 14 -\Italic -\Justification Center -\SpaceBefore 16 -\SpaceAfter 4 - -\Marker mr -\Name mr - Heading - Major Section Range References -\Description A major section division references range heading (basic) -\TextType Section -\TextProperties paragraph publishable vernacular level_1 -\StyleType Paragraph -\FontSize 12 -\Italic -\Justification Center -\SpaceAfter 4 - -\Marker s -\Name s - Heading - Section Level 1 -\Description A section heading, level 1 (if single level) (basic) -\TextType Section -\TextProperties paragraph publishable vernacular level_1 -\StyleType Paragraph -\FontSize 12 -\Bold -\Justification Center -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker s1 -\Name s1 - Heading - Section Level 1 -\Description A section heading, level 1 (if multiple levels) -\TextType Section -\TextProperties paragraph publishable vernacular level_1 -\StyleType Paragraph -\FontSize 12 -\Bold -\Justification Center -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker s2 -\Name s2 - Heading - Section Level 2 -\Description A section heading, level 2 (e.g. Proverbs 22-24) -\TextType Section -\TextProperties paragraph publishable vernacular level_2 -\StyleType Paragraph -\FontSize 12 -\Italic -\Justification Center -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker s3 -\Name s3 - Heading - Section Level 3 -\Description A section heading, level 3 (e.g. Genesis "The First Day") -\TextType Section -\TextProperties paragraph publishable vernacular level_3 -\StyleType Paragraph -\FontSize 12 -\Italic -\Justification Left -\SpaceBefore 6 -\SpaceAfter 3 - -\Marker s4 -\Name s4 - Heading - Section Level 4 -\Description A section heading, level 4 -\TextType Section -\TextProperties paragraph publishable vernacular level_4 -\StyleType Paragraph -\FontSize 12 -\Italic -\Justification Left -\SpaceBefore 6 -\SpaceAfter 3 - -\Marker sr -\Name sr - Heading - Section Range References -\Description A section division references range heading -\TextType Section -\TextProperties paragraph publishable vernacular level_1 -\StyleType Paragraph -\FontSize 12 -\Bold -\Justification Center -\SpaceAfter 4 - -\Marker r -\Name r - Heading - Parallel References -\Description Parallel reference(s) (basic) -\TextType Section -\TextProperties paragraph publishable vernacular level_1 -\StyleType Paragraph -\FontSize 12 -\Italic -\Justification Center -\SpaceAfter 4 - -\Marker sp -\Name sp - Heading - Speaker -\Description A heading, to identify the speaker (e.g. Job) -\TextType Section -\TextProperties paragraph publishable vernacular level_1 -\StyleType Paragraph -\FontSize 12 -\Italic -\Justification Left -\SpaceBefore 8 -\SpaceAfter 4 - -\Marker d -\Name d - Heading - Descriptive Title - Hebrew Subtitle -\Description A Hebrew text heading, to provide description (e.g. Psalms) -\TextType VerseText -\TextProperties paragraph publishable vernacular level_1 -\StyleType Paragraph -\FontSize 12 -\Italic -\Justification Center -\SpaceBefore 4 -\SpaceAfter 4 - -# Tables - -\Marker tr -\Name tr - Table - Row -\Description A new table row -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\LeftMargin .5 -\FirstLineIndent -.25 # 6/16 inch - -\Marker th1 -\Name th1 - Table - Column 1 Heading -\Description A table heading, column 1 -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic - -\Marker th2 -\Name th2 - Table - Column 2 Heading -\Description A table heading, column 2 -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic - -\Marker th3 -\Name th3 - Table - Column 3 Heading -\Description A table heading, column 3 -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic - -\Marker th4 -\Name th4 - Table - Column 4 Heading -\Description A table heading, column 4 -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic - -\Marker th5 -\Name th5 - Table - Column 5 Heading -\Description A table heading, column 5 -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic - -\Marker tc1 -\Name tc1 - Table - Column 1 Cell -\Description A table cell item, column 1 -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 - -\Marker tc2 -\Name tc2 - Table - Column 2 Cell -\Description A table cell item, column 2 -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 - -\Marker tc3 -\Name tc3 - Table - Column 3 Cell -\Description A table cell item, column 3 -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 - -\Marker tc4 -\Name tc4 - Table - Column 4 Cell -\Description A table cell item, column 4 -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 - -\Marker tc5 -\Name tc5 - Table - Column 5 Cell -\Description A table cell item, column 5 -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 - -# Center Aligned Table Heads and Columns - -\Marker thc1 -\Name thc1 - Table - Column 1 Heading - Center Aligned -\Description A table heading, column 1, center aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic -\Justification Center - -\Marker thc2 -\Name thc2 - Table - Column 2 Heading - Center Aligned -\Description A table heading, column 2, center aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic -\Justification Center - -\Marker thc3 -\Name thc3 - Table - Column 3 Heading - Center Aligned -\Description A table heading, column 3, center aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic -\Justification Center - -\Marker thc4 -\Name thc4 - Table - Column 4 Heading - Center Aligned -\Description A table heading, column 4, center aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic -\Justification Center - -\Marker thc5 -\Name thc5 - Table - Column 5 Heading - Center Aligned -\Description A table heading, column 5, center aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic -\Justification Center - -\Marker tcc1 -\Name tcc1 - Table - Column 1 Cell - Center Aligned -\Description A table cell item, column 1, center aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Justification Center - -\Marker tcc2 -\Name tcc2 - Table - Column 2 Cell - Center Aligned -\Description A table cell item, column 2, center aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Justification Center - -\Marker tcc3 -\Name tcc3 - Table - Column 3 Cell - Center Aligned -\Description A table cell item, column 3, center aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Justification Center - -\Marker tcc4 -\Name tcc4 - Table - Column 4 Cell - Center Aligned -\Description A table cell item, column 4, center aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Justification Center - -\Marker tcc5 -\Name tcc5 - Table - Column 5 Cell - Center Aligned -\Description A table cell item, column 5, center aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Justification Center - -# Right Aligned Table Heads and Columns - -\Marker thr1 -\Name thr1 - Table - Column 1 Heading - Right Aligned -\Description A table heading, column 1, right aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic -\Justification Right - -\Marker thr2 -\Name thr2 - Table - Column 2 Heading - Right Aligned -\Description A table heading, column 2, right aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic -\Justification Right - -\Marker thr3 -\Name thr3 - Table - Column 3 Heading - Right Aligned -\Description A table heading, column 3, right aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic -\Justification Right - -\Marker thr4 -\Name thr4 - Table - Column 4 Heading - Right Aligned -\Description A table heading, column 4, right aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic -\Justification Right - -\Marker thr5 -\Name thr5 - Table - Column 5 Heading - Right Aligned -\Description A table heading, column 5, right aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic -\Justification Right - -\Marker tcr1 -\Name tcr1 - Table - Column 1 Cell - Right Aligned -\Description A table cell item, column 1, right aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Justification Right - -\Marker tcr2 -\Name tcr2 - Table - Column 2 Cell - Right Aligned -\Description A table cell item, column 2, right aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Justification Right - -\Marker tcr3 -\Name tcr3 - Table - Column 3 Cell - Right Aligned -\Description A table cell item, column 3, right aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Justification Right - -\Marker tcr4 -\Name tcr4 - Table - Column 4 Cell - Right Aligned -\Description A table cell item, column 4, right aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Justification Right - -\Marker tcr5 -\Name tcr5 - Table - Column 5 Cell - Right Aligned -\Description A table cell item, column 5, right aligned -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Justification Right - -# Lists - -\Marker li -\Name li - List Entry - Level 1 -\Description A list entry, level 1 (if single level) -\TextType VerseText -\TextProperties paragraph publishable vernacular level_1 -\StyleType paragraph -\FontSize 12 -\LeftMargin .5 -\FirstLineIndent -.375 # 1/8 inch indent, 1/2 inch wrap - -\Marker li1 -\Name li1 - List Entry - Level 1 -\Description A list entry, level 1 (if multiple levels) -\TextType VerseText -\TextProperties paragraph publishable vernacular level_1 -\StyleType paragraph -\FontSize 12 -\LeftMargin .5 -\FirstLineIndent -.375 # 1/8 inch indent, 1/2 inch wrap - -\Marker li2 -\Name li2 - List Entry - Level 2 -\Description A list entry, level 2 -\TextType VerseText -\TextProperties paragraph publishable vernacular level_2 -\StyleType paragraph -\FontSize 12 -\LeftMargin .75 -\FirstLineIndent -.375 - -\Marker li3 -\Name li3 - List Entry - Level 3 -\Description A list entry, level 3 -\TextType VerseText -\TextProperties paragraph publishable vernacular level_3 -\StyleType paragraph -\FontSize 12 -\LeftMargin 1 -\FirstLineIndent -.375 - -\Marker li4 -\Name li4 - List Entry - Level 4 -\Description A list entry, level 4 -\TextType VerseText -\TextProperties paragraph publishable vernacular level_4 -\StyleType paragraph -\FontSize 12 -\LeftMargin 1.25 -\FirstLineIndent -.375 - -# Footnotes - -\Marker f -\Endmarker f* -\Name f...f* - Footnote -\Description A Footnote text item (basic) -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Note -\FontSize 12 - -\Marker fe -\Endmarker fe* -\Name fe...fe* - Endnote -\Description An Endnote text item -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Note -\FontSize 12 - -\Marker fr -\Endmarker fr* -\Name fr - Footnote - Reference -\Description The origin reference for the footnote (basic) -\TextType NoteText -\TextProperties publishable vernacular note -\StyleType Character -\FontSize 12 -\Bold - -\Marker ft -\Endmarker ft* -\Name ft - Footnote - Text -\Description Footnote text, Protocanon (basic) -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 - -\Marker fk -\Endmarker fk* -\Name fk - Footnote - Keyword -\Description A footnote keyword (basic) -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 -\Bold -\Italic - -\Marker fq -\Endmarker fq* -\Name fq - Footnote - Quotation or Alternate Rendering -\Description A footnote scripture quote or alternate rendering (basic) -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 -\Italic - -\Marker fqa -\Endmarker fqa* -\Name fqa - Footnote - Alternate Translation Rendering -\Description A footnote alternate rendering for a portion of scripture text -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 -\Italic - -\Marker fl -\Endmarker fl* -\Name fl - Footnote - Label Text -\Description A footnote label text item, for marking or "labelling" the type or alternate translation being provided in the note. -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 -\Italic -\Bold - -\Marker fp -\Endmarker fp* -\Name fp - Footnote Paragraph Mark -\Description A Footnote additional paragraph marker -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 - -\Marker fv -\Endmarker fv* -\Name fv...fv* - Footnote - Embedded Verse Number -\Description A verse number within the footnote text -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 -\Superscript - -\Marker fdc -\Endmarker fdc* -\Name fdc...fdc* - Footnote - DC text -\Description Footnote text, applies to Deuterocanon only -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 - -\Marker fm -\Endmarker fm* -\Name fm - Footnote - Additional Caller to Previous Note -\Description An additional footnote marker location for a previous footnote -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Character -\FontSize 12 -\Superscript - -# Cross References - -\Marker x -\Endmarker x* -\Name x...x* - Cross Reference -\Description A list of cross references (basic) -\TextProperties publishable vernacular note crossreference -\TextType NoteText -\StyleType Note -\FontSize 12 - -\Marker xo -\Endmarker xo* -\Name xo - Cross Reference - Origin Reference -\Description The cross reference origin reference (basic) -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 -\Bold - -\Marker xt -\Endmarker xt* -\Name xt - Cross Reference - Target References -\Description The cross reference target reference(s), protocanon only (basic) -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 - -\Marker xk -\Endmarker xk* -\Name xk - Cross Reference - Keyword -\Description A cross reference keyword -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 -\Italic - -\Marker xq -\Endmarker xq* -\Name xq - Cross Reference - Quotation -\Description A cross-reference quotation from the scripture text -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 -\Italic - -\Marker xot -\Endmarker xot* -\Name xot...xot* - Cross Reference - OT Target Refs (optional) -\Description Cross-reference target reference(s), Old Testament only -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 - -\Marker xnt -\Endmarker xnt* -\Name xnt...xnt* - Cross Reference - NT Target Refs (optional) -\Description Cross-reference target reference(s), New Testament only -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 - -\Marker xdc -\Endmarker xdc* -\Name xdc...xdc* - Cross Reference - DC Target Refs -\Description Cross-reference target reference(s), Deuterocanon only -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 - -\Marker rq -\Endmarker rq* -\Name rq...rq* - Cross Reference - Inline Quotation References -\Description A cross-reference indicating the source text for the preceding quotation. -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 10 -\Italic - -# Other Special Text - -\Marker qt -\Endmarker qt* -\Name qt...qt* - Special Text - Quoted Text - OT in NT -\Description For Old Testament quoted text appearing in the New Testament (basic) -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic - -\Marker nd -\Endmarker nd* -\Name nd...nd* - Special Text - Name of Deity -\Description For name of deity (basic) -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Underline - -\Marker tl -\Endmarker tl* -\Name tl...tl* - Special Text - Transliterated Word -\Description For transliterated words -\TextType VerseText -\TextProperties publishable nonvernacular -\StyleType Character -\FontSize 12 -\Italic - -\Marker dc -\Endmarker dc* -\Name dc...dc* - Special Text - Deuterocanonical/LXX Additions -\Description Deuterocanonical/LXX additions or insertions in the Protocanonical text -\TextProperties publishable vernacular -\StyleType Character -\Italic - -\Marker bk -\Endmarker bk* -\Name bk...bk* - Special Text - Quoted book title -\Description For the quoted name of a book -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic - -\Marker sig -\Endmarker sig* -\Name sig...sig* - Special Text - Author's Signature (Epistles) -\Description For the signature of the author of an Epistle -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic - -\Marker pn -\Endmarker pn* -\Name pn...pn* - Special Text - Proper Name -\Description For a proper name -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Bold -\Underline - -\Marker addpn -\Endmarker addpn* -\Name addpn...addpn* - Special Text for Chinese -\Description For chinese words to be dot underline & underline -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -#\Color 2263842 -\Bold -\Italic -\Underline - -\Marker wj -\Endmarker wj* -\Name wj...wj* - Special Text - Words of Jesus -\Description For marking the words of Jesus -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Color 255 - -\Marker k -\Endmarker k* -\Name k...k* - Special Text - Keyword -\Description For a keyword -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic -\Bold - -\Marker sls -\Endmarker sls* -\Name sls...sls* - Special Text - Secondary Language or Text Source -\Description To represent where the original text is in a secondary language or from an alternate text source -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic - -\Marker ord -\Endmarker ord* -\Name ord...ord* - Special Text - Ordinal number text portion -\Description For the text portion of an ordinal number -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Superscript - -\Marker add -\Endmarker add* -\Name add...add* - Special Text - Translational Addition -\Description For a translational addition to the text -\TextType VerseText -\TextProperties publishable vernacular -\StyleType Character -#\Color 2263842 -\Bold -\Italic - -\Marker lit -\Name lit - Special Text - Liturgical note -\Description For a comment or note inserted for liturgical use -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\Justification Right -\FontSize 12 -\Bold - -# Character Styling - -\Marker no -\Endmarker no* -\Name no...no* - Character - Normal Text -\Description A character style, use normal text -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 - -\Marker it -\Endmarker it* -\Name it...it* - Character - Italic Text -\Description A character style, use italic text -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic - -\Marker bd -\Endmarker bd* -\Name bd...bd* - Character - Bold Text -\Description A character style, use bold text -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Bold - -\Marker bdit -\Endmarker bdit* -\Name bdit...bdit* - Character - BoldItalic Text -\Description A character style, use bold + italic text -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Bold -\Italic - -\Marker em -\Endmarker em* -\Name em...em* - Character - Emphasized Text -\Description A character style, use emphasized text style -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Italic - -\Marker sc -\Endmarker sc* -\Name sc...sc* - Character - Small Caps -\Description A character style, for small capitalization text -\TextProperties publishable vernacular -\StyleType Character -\FontSize 12 -\Smallcaps - -# Breaks - -\Marker pb -\Name pb - Break - Page Break -\Description Page Break used for new reader portions and children's bibles where content is controlled by the page -\TextType Other -\TextProperties publishable -\StyleType Paragraph -\FontSize 12 - -# Special Features - -\Marker fig -\Endmarker fig* -\Name fig...fig* - Auxiliary - Figure/Illustration/Map -\Description Illustration [Columns to span, height, filename, caption text] -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Character -\FontSize 12 - -\Marker pro -\Endmarker pro* -\Name pro...pro* - Special Text - CJK Pronunciation -\Description For indicating pronunciation in CJK texts -\TextType Other -\TextProperties Nonpublishable -\StyleType Character -\FontSize 10 -#\Color 128 - -# Peripheral References - -\Marker w -\Endmarker w* -\Name w...w* - Peripheral Ref - Wordlist Entry -\Description A wordlist text item -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Character -\FontSize 12 - -\Marker wh -\Endmarker wh* -\Name wh...wh* - Peripheral Ref - Hebrew Wordlist Entry -\Description A Hebrew wordlist text item -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Character -\FontSize 12 - -\Marker wg -\Endmarker wg* -\Name wg...wg* - Peripheral Ref - Greek Wordlist Entry -\Description A Greek Wordlist text item -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Character -\FontSize 12 - -\Marker ndx -\Endmarker ndx* -\Name ndx...ndx* - Peripheral Ref - Subject Index Entry -\Description A subject index text item -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Character -\FontSize 12 - -# Peripheral Materials -# Content Division Marker - -\Marker periph -\Name periph - Peripherals - Content Division Marker -\Description Periheral content division marker which should be followed by an additional division argument/title. -\TextType Section -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 14 -\Bold -\SpaceBefore 16 -\SpaceAfter 4 -\Color 33023 - -# Additional peripheral material extensions to existing USFM markup. - -\Marker p1 -\Name p1 - Periph - Front/Back Matter Paragraph Level 1 -\Description Front or back matter text paragraph, level 1 (if multiple levels) -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\FirstLineIndent .125 # 1/8 inch first line indent - -\Marker p2 -\Name p2 - Periph - Front/Back Matter Paragraph Level 2 -\Description Front or back matter text paragraph, level 2 (if multiple levels) -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\FirstLineIndent .125 # 1/4 inch first line indent -\LeftMargin .125 - -\Marker k1 -\Name k1 - Periph - Concordance Keyword Level 1 -\Description Concordance main entry text or keyword, level 1 -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 - -\Marker k2 -\Name k2 - Periph - Concordance Keyword Level 2 -\Description Concordance main entry text or keyword, level 2 -\TextType Other -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 - -\Marker xtSee -\Endmarker xtSee* -\Name xtSee - Concordance and Names Index - Alternate Entry Target Reference -\Description Concordance and Names Index markup for an alternate entry target reference. -\TextProperties publishable vernacular -\TextType NoteText -\StyleType Character -\FontSize 12 -\Italic -\Color 16711680 - -\Marker xtSeeAlso -\Endmarker xtSeeAlso* -\Name xtSeeAlso - Concordance and Names Index - Additional Entry Target Reference -\Description Concordance and Names Index markup for an additional entry target reference. -\TextProperties publishable vernacular -\TextType Other -\StyleType Character -\FontSize 12 -\Italic -\color 16711680 - -# Other special text elements specified in USFM -# ~ = fixed (no-break) space -# // = discretionary line break - -# Obsolete, deprecated, or no longer officially part of USFM. -# These markers may have existed in earlier resource and stylesheet revisions. - -\Marker pr -\Name DEPRECATED pr - Paragraph - Right Aligned -\Description Paragraph text, right aligned -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\Justification Right -\FontSize 12 - -\Marker ph -\Name DEPRECATED ph - Paragraph - Hanging Indent - Level 1 -\Description Paragraph text, with level 1 hanging indent (if single level) -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\LeftMargin .5 -\FirstLineIndent -.25 # 1/4 inch indent, 1/2 inch wrap -\FontSize 12 - -\Marker ph1 -\Name DEPRECATED ph1 - Paragraph - Hanging Indent - Level 1 -\Description Paragraph text, with level 1 hanging indent (if multiple levels) -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\LeftMargin .5 -\FirstLineIndent -.25 # 1/4 inch indent, 1/2 inch wrap -\FontSize 12 - -\Marker ph2 -\Name DEPRECATED ph2 - Paragraph - Hanging Indent - Level 2 -\Description Paragraph text, with level 2 hanging indent -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\LeftMargin .75 -\FirstLineIndent -.25 # 1/2 inch indent, 3/4 inch wrap -\FontSize 12 - -\Marker ph3 -\Name DEPRECATED ph3 - Paragraph - Hanging Indent - Level 3 -\Description Paragraph text, with level 3 hanging indent -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\LeftMargin 1 -\FirstLineIndent -.25 # 3/4 inch indent, 1 inch wrap -\FontSize 12 - -\Marker phi -\Name DEPRECATED phi - Paragraph - Indented - Hanging Indent -\Description Paragraph text, indented with hanging indent -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\LeftMargin 1 - -\Marker tr1 -\Name OBSOLETE tr1 - Table - Row - Level 1 -\Description A table Row -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\LeftMargin .5 -\FirstLineIndent -.25 # 6/16 inch - -\Marker tr2 -\Name OBSOLETE tr2 - Table - Row - Level 2 -\Description A table Row -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\LeftMargin .75 -\FirstLineIndent -.25 # 6/16 inch - -\Marker ps -\Name OBSOLETE ps - Paragraph - No Break with Next Paragraph -\Description Paragraph text, no break with next paragraph text at chapter boundary -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FontSize 12 -\FirstLineIndent .125 # 1/8 inch first line indent - -\Marker psi -\Name OBSOLETE psi - Paragraph - Indented - No Break with Next -\Description Paragraph text, indented, with no break with next paragraph text (at chapter boundary) -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Paragraph -\FirstLineIndent .125 # 1/8 inch first line indent -\LeftMargin .25 -\RightMargin .25 -\FontSize 12 - -\Marker fs -\Endmarker fs* -\Name DEPRECATED fs - Footnote - Footnote Summary -\Description A summary text for the concept/idea/quotation from the scripture translation for which the note is being provided. -\TextProperties publishable vernacular note -\TextType NoteText -\StyleType Character -\FontSize 12 -\Italic - -\Marker wr -\Endmarker wr* -\Name OBSOLETE wr...wr* - Auxiliary - Wordlist/Glossary Reference -\Description A Wordlist text item -\TextProperties publishable vernacular -\TextType VerseText -\StyleType Character -\FontSize 12 -\Italic - -# 2.0x peripheral markup (replaced with \periph + Content Division Title/Argument) - -\Marker pub -\Name OBSOLETE pub Peripherals - Front Matter Publication Data -\Description Front matter publication data -\TextProperties paragraph publishable vernacular poetic -\TextType VerseText -\FontSize 10 -\StyleType paragraph - -\Marker toc -\Name OBSOLETE toc Peripherals - Front Matter Table of Contents -\Description Front matter table of contents -\TextProperties paragraph publishable vernacular poetic -\TextType VerseText -\FontSize 10 -\StyleType paragraph - -\Marker pref -\Name OBSOLETE pref Peripherals - Front Matter Preface -\Description Front matter preface -\TextProperties paragraph publishable vernacular poetic -\TextType VerseText -\FontSize 10 -\StyleType paragraph - -\Marker intro -\Name OBSOLETE intro Peripherals - Front Matter Introduction -\Description Front matter introduction -\TextProperties paragraph publishable vernacular poetic -\TextType VerseText -\FontSize 10 -\StyleType paragraph - -\Marker conc -\Name OBSOLETE conc Peripherals - Back Matter Concordance -\Description Back matter concordance -\TextProperties paragraph publishable vernacular poetic -\TextType VerseText -\FontSize 10 -\StyleType paragraph - -\Marker glo -\Name OBSOLETE glo Peripherals - Back Matter Glossary -\Description Back matter glossary -\TextProperties paragraph publishable vernacular poetic -\TextType VerseText -\FontSize 10 -\StyleType paragraph - -\Marker idx -\Name OBSOLETE idx Peripherals - Back Matter Index -\Description Back matter index -\TextProperties paragraph publishable vernacular poetic -\TextType VerseText -\FontSize 10 -\StyleType paragraph - -\Marker maps -\Name OBSOLETE maps Peripherals - Back Matter Map Index -\Description Back matter map index -\TextProperties paragraph publishable vernacular poetic -\TextType VerseText -\FontSize 10 -\StyleType paragraph - -\Marker cov -\Name OBSOLETE cov Peripherals - Other - Cover -\Description Other peripheral materials - cover -\TextProperties paragraph publishable vernacular poetic -\TextType VerseText -\FontSize 10 -\StyleType paragraph - -\Marker spine -\Name OBSOLETE spine Peripherals - Other - Spine -\Description Other peripheral materials - spine -\TextProperties paragraph publishable vernacular poetic -\TextType VerseText -\FontSize 10 -\StyleType paragraph - -\Marker pubinfo -\Name OBSOLETE pubinfo - Publication - Information -\Description Publication information - Lang,Credit,Version,Copies,Publisher,Id,Logo -\TextType Other -\TextProperties paragraph nonpublishable nonvernacular -\StyleType Paragraph -\FontSize 12 -\Color 16711680 - -# Concordance/Names Index Tools - special sfms for use in Publishing Assistant - -\Marker zpa-xb -\Endmarker zpa-xb* -\Name zpa-xb - Periph - Book -\Description Book Ref -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Character -\FontSize 12 - -\Marker zpa-xc -\Endmarker zpa-xc* -\Name zpa-xc - Periph - Chapter -\Description Chapter Ref -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Character -\FontSize 12 -\Bold - -\Marker zpa-xv -\Endmarker zpa-xv* -\Name zpa-xv - Periph - Verse -\Description Verse Ref -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Character -\FontSize 12 - -\Marker zpa-d -\Endmarker zpa-d* -\Name zpa-d - Periph - Description -\Description Description -\TextType VerseText -\TextProperties paragraph publishable vernacular -\StyleType Character -\FontSize 12 diff --git a/samples/data/en-hc.xml b/samples/data/en-hc.xml deleted file mode 100644 index 96d7419fb..000000000 --- a/samples/data/en-hc.xml +++ /dev/null @@ -1,265 +0,0 @@ - - - - - English - - - Noun - - - Verb - - - Article - - - - English Orthography - - - - a - - - - - e - - - - - i - - - - - o - - - - - u - - - - - y - - - - - - b - - - - - c - - - - - d - - - - - f - - - - - g - - - - - h - - - - - j - - - - - k - - - - - l - - - - - m - - - - - n - - - - - p - - - - - q - - - - - r - - - - - s - - - - - t - - - - - v - - - - - w - - - - - x - - - - - z - - - - - - - + - - - - - - - Any - - - - - morphophonemic - - - Past tense - - - - - - - - - - - - - +ed - - - - - rule1 - PAST - - - Plural - - - - - - - - - - - - - +s - - - - - rule2 - PLURAL - - - - - - - god - - - le1 - god - - - - - creat - - - create - - - - - world - - - world - - - - - the - - - le2 - the - - - - - bill - - - le5 - bill - - - - - surface - - - - \ No newline at end of file diff --git a/samples/data/en.txt b/samples/data/en.txt deleted file mode 100644 index 739411113..000000000 --- a/samples/data/en.txt +++ /dev/null @@ -1,1000 +0,0 @@ -Would you mind giving us the keys to the room, please? -I have made a reservation for a quiet, double room with a telephone and a tv for Rosario Cabedo. -Would you mind moving me to a quieter room? -I have booked a room. -I think that there is a problem. -Do you have any rooms with a tv, air conditioning and a safe available? -Would you mind showing us a room with a tv? -Does it have a telephone? -I am leaving on the second at eight in the evening. -How much does a single room cost per week? -Do you accept cash? -My name is Victoria Piquer. -Would you mind sending up my luggage to room number three one oh? -Do you accept cash? -There is a mistake in my bill. -Could you explain the bill for me, please? -I want to book a double room for a night. -Does it have a telephone? -Could you wake us up at five o'clock, please? -I should leave on Tuesday March the seventh in the morning. -We have made a reservation. -Could you wake us up at twelve, please? -Can you send our luggage to the taxi? -I have a reservation for Julia Calatayud. -I have booked a room. -A room with a telephone and a view for Mrs Ramírez, please. -Can you send up our bags to the room, please? -I have booked a room for Mr Roda. -How much does a double room with half board cost per day? -Are all expenses included in the bill? -I think that there is a mistake. -We want you to send up our suitcase to the room, please. -I would like a room. -Prepare the bill. -I want a quiet, double room, please. -Please make out the bill. -We want you to send our luggage to our room, please. -Could you wake us up at seven, please? -Could we see a room? -Could you wake us up tomorrow at a quarter to nine, please? -Put my suitcases on the bus. -I want a room until the twentieth, please. -Could we book two rooms? -Is there a quiet room available? -I have booked a quiet room with a tv and a good view of the mountain for Rosario Cantero. -Do you have a double room available? -I made a reservation. -Is the phone bill included? -Can you give us the key to room number two nine four, please? -Could you call a taxi, please? -The room is very cold. -I have made a reservation. -Would you mind showing us another room? -We would like you to send up our suitcases to our room. -Can you change our room? -Will you call a taxi for me, please? -The room is too hot. -Would you mind sending up my travel bags to the room? -I would like to book a room with air conditioning for twenty days, please. -Could I see a quiet room, please? -Wake me up at a quarter past two. -Could we see a room, please? -I have booked a room until next Friday. -I am leaving on Friday. -Give me the key to room number one four nine, please. -I am leaving today at eight in the morning. -I have made a reservation for a room with a good view of the mountain and a bathroom for Pilar Jiménez. -Please check the bill. -Do you have a room with a tv, air conditioning and a minibar available? -Could you wake me up tomorrow at two o'clock? -We have booked a double room until the day after tomorrow. -Could you call a taxi for me, please? -I would like you to wake me up tomorrow at four, please. -I want a room for a night, please. -Could you ask for a taxi for us, please? -I want to book a single room for twenty-five nights. -I have a reservation. -What is the date? -Please make out the bill. -I have booked a single room. -I booked a room for Mr and Mrs Tena. -We have made a reservation. -Is everything included? -Could you call a taxi for us, please? -Carry our luggage. -Would you mind giving us the key to the room? -I made a reservation for a double room for Miguel Ramírez. -We have booked a quiet room. -Can you give me the key to the room, please? -Is the phone bill included? -I made a reservation. -I want a room with air conditioning for today. -Would you mind changing my room? -We have booked two rooms with a telephone. -I booked a double room with a telephone for today. -Is there air conditioning in my room, please? -I have booked a room. -Would you mind sending the travel bags to the station? -I have booked a quiet, double room for Enrique Velasco. -Could you send my bags down? -Would you mind giving us the key to our room, please? -We want a double room for a night, please. -I want to move to a room with a tv, please. -Do you have a quiet room available? -I have booked a room with a telephone. -Please send my luggage to my room. -Is the phone bill included? -Send my luggage to the station. -Would you call our taxi for us, please? -How much does a single room with half board cost? -The room is too cold. -Could you send the luggage to the bus? -We made a reservation. -Could you make out our bill, please? -I have booked a quiet, double room for Silvia Quereda. -My name is Jesús Rivera. -I have made a reservation for Sergio Moliner. -Do you have a room available? -Could you ask for a taxi for me, please? -Could you prepare our bill for room number six oh nine for us, please? -Would you mind sending down our luggage to the room? -Could you ask for a taxi for me, please? -I would like to book a room until the twenty-fifth. -I want to book a double room with a telephone, please. -Could you wake me up at two? -Can you wake us up at eleven, please? -Do you have a room available? -The room is too cold. -Could you call a taxi for me, please? -How much is a single room with full board for a day? -Could you wake me up at one o'clock, please? -I have a reservation for a quiet, double room with a tv and a view. -Could you send our suitcases to room number eight one nine, please? -Sorry. -I have booked a quiet, double room with a tv and a telephone. -Could you wake me up at a quarter to eleven? -I should leave this afternoon. -There has been a mistake in the bill. -We would like to change rooms, please. -I think that there was a mistake in the bill for room number eight seven four. -How much is a single room including breakfast, please? -Could you wake me up at half past three, please? -The room is very cold. -Would you mind giving me the key to the room? -I have booked a room. -I would like to change rooms. -My name is Sergio Iborra. -I have made a reservation for Mr Miralles. -I have a reservation for Mr Ortiz. -We should leave on Saturday March the seventh. -Can you wake us up at half past ten, please? -I would like you to send down our luggage to our room, please. -How much is a single room for a night? -How much is a double room per day, please? -I want a double room. -I want you to give me the key to room number three three three. -I booked a room. -Is breakfast included? -The room is very expensive. -We are leaving on September the thirteenth at a quarter to eight in the morning. -Can you wake me up at half past eleven, please? -Would you mind sending my bags to the bus? -We are leaving tomorrow at four. -I think that there has been an error in the bill. -Would you mind me taking a look at a room? -I want you to send down our luggage to the room, please. -Could you give me the keys to room number six one three, please? -How much does a double room including room service cost, please? -Can you explain the bill for room number oh one four for me, please? -The room is all right. -I made a reservation. -I have made a reservation for a quiet, double room with a telephone and a shower. -We should leave tomorrow evening. -Are there a tv and air conditioning in the room? -Wake me up tomorrow at half past ten, please. -Please give me the key to room number nine one oh. -We have booked a double room with a bath for today. -Does it have air conditioning, a telephone and a tv? -A mistake has been made in our bill. -Could I see a single room with air conditioning? -How much is a double room? -Could I see a double room, please? -Is there a quiet room available? -I made a reservation for a room with a tv and a telephone for Roberto Arnau. -I have booked a room with a tv and a bath for Gerardo Ródenas. -I would like a room, please. -We have booked a room. -We are leaving on Thursday September the thirtieth at half past four. -Could I see a double room with a tv? -Is everything included? -Could you give us the key to the room, please? -Can you give me the key to room number seven seven three? -Would you make out the bill for us, please? -We should leave on Thursday April the ninth in the afternoon. -I am leaving today at six in the morning. -I booked a quiet, double room. -I booked a quiet room for Amelia Peinado. -Would you ask for a taxi for room number nine eight one for me, please? -How much does a double room cost, please? -Would you ask for a taxi for us, please? -Could you explain our bill for room number nine four one, please? -Is everything included? -Can you prepare my bill for me, please? -I have made a reservation for a quiet, double room with a view, a shower and a telephone for Carmelo Peris. -Could you give me the key to the room, please? -Could we see another room, please? -Can you carry the travel bags? -I would like to move to a quieter room, please. -Could you wake us up tomorrow at a quarter past nine? -I want to book a room, please. -Could we book two double rooms for fifteen days? -It does not matter! -Could we book two rooms? -Do you have a double room available? -Please prepare the bill. -I would like a room, please. -Is room service included? -We are leaving this evening. -Send up our luggage to our room. -My name is Federico Viciano. -I have booked a quiet room. -I have a reservation for Mrs Guijarro. -Would you mind waking us up at half past one? -Do you have any quiet, single rooms available? -We should leave tomorrow. -I have booked a room. -Could you make out the bill for me, please? -Could I see a quiet, double room, please? -A mistake has been made in the bill. -I have made a reservation for a room with a tv, a telephone and a view of the sea for Asunción Espinosa. -Does the room have air conditioning and hot water? -I want to book a room until next Saturday. -Could you call a taxi for room number three one three for us, please? -I want a room. -Can you wake me up at a quarter to one? -A room, please. -I think that there was an error in the bill. -I would like a room for five nights. -We have booked a single room with a telephone for Mr and Mrs Santos. -Are all taxes included? -Is the phone bill included? -Please send our travel bag to room number five two seven. -I have booked a quiet room. -Ask for a taxi. -Can you give us the key to the room? -Could you send the bags to the car? -I have made a reservation for Susana Padilla. -Can you send down our luggage to the room, please? -Would you mind sending down our luggage to the room, please? -How much does a double room cost per week, please? -Wake me up at half past one, please. -We should leave tomorrow at six. -I have made a reservation. -The room is too noisy. -I want you to give us the keys to room number seven one six, please. -Is everything included? -I have booked a double room. -I would like to book a single room until the twenty-first, please. -Can you wake me up tomorrow at a quarter to three, please? -Wake us up at one, please. -I should leave on Monday July the second in the afternoon. -Should I fill in a registration form, please? -Could I see a room? -I think that there was an error in the bill. -The room is very expensive. -We would like you to give us the keys to room number four three, please. -Could you explain the bill for us, please? -Could you give us the keys to room number two two six, please? -How much does a single room cost, please? -Could we book two rooms for tonight? -Could you wake us up at one, please? -I think that there is a problem in my bill for room number nine three three. -No! -My name is Marina Varela. -I want to change rooms, please. -The keys to room number eight two oh, please. -Are expenses included in the bill? -I leave today. -Would you mind giving us the keys to the room? -Do the rooms have air conditioning, please? -Can you put the luggage in the car? -Would you mind giving us the key to the room? -Could you send down my suitcases to my room? -I want a room with a telephone, a view, a tv and a bath for Marina Sáez. -I have a reservation for a quiet, double room with a view of the town and a bath for Ricardo Salsas. -Should we sign the registration form, please? -Could you wake us up tomorrow at half past six? -I should leave on Friday April the nineteenth at nine in the evening. -I think that a mistake has been made in the bill. -I would like to change rooms, please. -Is the phone bill included? -Can you wake me up at a quarter to two, please? -We are leaving on the seventh at half past eleven in the evening. -I would like you to wake us up at half past three, please. -We should leave tomorrow at nine. -Is there a minibar in the room? -We booked a room until next Thursday. -Would you mind showing me a double room with a tv, please? -I think that there is an error in the bill for room number eight three eight. -Could you make out the bill for room number nine oh one for us, please? -I want to book a quiet, single room until November the first, please. -I have booked a quiet, single room with a view of the mountain and a tv for Mrs Gómez. -I have made a reservation. -Could I book a room? -We are leaving on Monday April the nineteenth in the morning. -I have booked a quiet room. -Could you make out my bill for me, please? -I should leave today at half past eleven in the morning. -Could you prepare the bill, please? -Do you have a room available? -I think that a mistake has been made in our bill for room number five one five. -I want to book a room, please. -Do you have a room available? -Are extras included in the bill? -I have booked a room. -We have made a reservation. -Can you ask for my taxi for room number seven one six for me, please? -Can you wake me up at a quarter past four? -Are all taxes included in the bill? -Is breakfast included? -I booked a room for Mr Gerardo Paches. -I have booked a single room for Mr Fernández. -I leave on the eighteenth at a quarter to seven in the morning. -Could you give us the key to room number six four seven, please? -Could we see a room, please? -Can you send our luggage to the bus? -I leave on Friday the first at three. -Can you send down my luggage to the room? -How much is a double room per day, please? -There is a lot of noise in the room. -Is the phone bill included? -The room is all right. -Is there hot water in the rooms, please? -Would you mind changing my room? -Could you put my travel bags in the taxi? -We would like you to wake us up at a quarter to eight, please. -It is very hot in the room. -We want to change rooms. -Put my bags on the bus. -Does my room have air conditioning? -Wake me up at six. -I am leaving tomorrow at a quarter to three in the afternoon. -Are there any rooms available? -Can you move us to a room with a telephone and a tv? -It does not matter. -Are there a telephone and a tv in the room? -I should leave today. -We are leaving this morning. -Would you mind showing us a room with a telephone and a tv, please? -Carry the luggage. -Is there air conditioning in the rooms? -Could you send up my luggage to my room? -I do not agree. -I have booked a single room. -I want to book a room for a day, please. -Could you give us the key to my room? -Can you give me the key to the room, please? -I have made a reservation. -I should leave tomorrow morning. -Should I fill in the registration form, please? -Would you mind changing my room? -Can you give me the keys to room number oh three oh, please? -Can you send down our travel bags to room number four two three, please? -I want to book a quiet, single room for a week, please. -Please explain the bill. -Could you ask for a taxi for room number seven two nine for us, please? -I have booked a room for tonight for Manuela Valls. -I would like you to give us the key to room number one one, please. -Can you send down our luggage to room number four one four, please? -I would like to change rooms, please. -We have booked a quiet, single room. -Please send up my travel bag to room number six three one. -Can you ask for a taxi, please? -Can you ask for a taxi, please? -Could you make out the bill for room number six two six, please? -Give us the key to room number eight one four. -I want you to send up my luggage to the room, please. -Give me the key to room number six one eight, please. -Could you give us the key to room number six four oh? -Could we see a quiet room, please? -Yes! -Send our luggage to room number eight two six, please. -Could you send down our luggage to room number three six two? -I have booked a quiet room until tomorrow for Gregorio Cabedo. -We are leaving today at half past six in the afternoon. -Would you mind sending our travel bags to room number two three five, please? -We made a reservation. -Could you explain the bill for me, please? -I would like to move to a room with a minibar and air conditioning, please. -We would like you to wake us up at a quarter past eleven. -We would like you to wake us up at nine o'clock. -I booked a room. -Can you move us to a room with a tv? -I should leave tomorrow at ten o'clock in the morning. -Can you explain the bill, please? -Please send up my luggage to my room. -Can you send our luggage to the bus? -Should we sign the registration form? -I would like you to wake me up tomorrow at ten, please. -We booked a double room. -We are leaving tomorrow at five o'clock in the afternoon. -Is there air conditioning in the room? -Would you mind giving me the key to room number three three three, please? -I am leaving tomorrow at half past four in the afternoon. -We should leave on Monday April the fourth at half past twelve in the morning. -What is the price for a double room with half board per day, please? -It is too hot in the room. -How much does a double room cost? -Would you mind showing us a room with a telephone, please? -How much does a single room cost per day? -I should leave. -Could you carry my bags? -How much does a double room cost for eighteen days, please? -Can you wake us up tomorrow at five o'clock? -Are all expenses included in the bill? -I have made a reservation for a double room with a bath and a telephone for Paloma Betoret. -I would like you to give me the keys to room number two three nine. -Could you check the bill for me, please? -I want you to send up my luggage to my room, please. -Can you give us the key to room number one four oh, please? -Would you mind giving me the keys to the room, please? -Could you wake us up at a quarter to six? -How much is a single room, please? -Could we book two quiet rooms for tonight? -I have booked a quiet, single room. -The room is very expensive. -I think that there was a problem in the bill. -I want a double room for today. -I want a single room. -I booked a room for Alicia Morales. -Okay. -Please give me the key to our room. -We want a room, please. -I would like to book two double rooms, please. -I am leaving tomorrow at a quarter to ten in the evening. -I want to change rooms. -The room is very hot. -I made a reservation for a quiet, single room with a good view of the sea, a telephone and a tv. -Would you mind changing our room? -We have booked a single room until the seventeenth. -I have a reservation. -Do you accept traveler checks? -We have booked a room. -How much? -How much is a single room per day, please? -Do you have a quiet, double room available? -Will you prepare the bill for us, please? -Can you wake us up at four, please? -We have booked a double room. -The room is all right. -Could you check the bill, please? -Please call my taxi. -I would like you to give us the key to my room. -I want you to wake us up at half past nine. -I would like a quiet, double room with a telephone, a good view of the mountain, a bath and a tv for Mrs Berrueco, please. -Could you ask for my taxi for me, please? -Could you ask for a taxi for me, please? -I think that there has been an error in the bill. -The room is too cold. -Could you call my taxi for me, please? -How much does a single room cost per week? -Would you mind me taking a look at a double room? -The room is all right. -Send our luggage to our room. -Could you carry my suitcases? -I want a room for twenty-nine days. -Should I fill in the registration form? -I want a room. -We want a single room until next Friday. -Give us the key to room number seven oh five. -Could you ask for a taxi, please? -We should leave on Saturday May the fourteenth. -Prepare our bill. -I should leave tomorrow at a quarter to five in the afternoon. -I want a room. -There is a mistake in the bill. -Do you have a single room available? -Is there a room available? -We have booked a single room for Luis Llopis. -We want a double room with a shower and air conditioning. -We have made a reservation. -Is the phone bill included? -Please give me the key to the room. -I think that a mistake has been made in our bill. -I have a reservation for a quiet, double room with a tv, a bathroom and a good view of the sea. -Would you mind waking me up at half past ten, please? -Please make out the bill. -I have booked a room for Jesús Barberá. -I want a room for a day, please. -Could you call a taxi for me, please? -I want you to send my luggage to my room, please. -Is there a tv in my room? -I booked a quiet room for twenty-six nights for Celestino Puig. -I leave on September the sixteenth at a quarter to twelve in the morning. -How much does a single room including breakfast cost per week? -Please send up our travel bag to room number one seven four. -Please prepare the bill. -I should leave on the third at a quarter past ten in the morning. -Do you have a room available? -I have booked a quiet room with a bathroom, a telephone and a tv. -Wake us up at a quarter past six. -Could we see a double room? -Could you wake us up at half past six? -I think that a mistake has been made in the bill. -I would like you to give me the key to the room, please. -Please ask for a taxi. -Do you have any rooms available? -We have booked a room for ten weeks for Rosario Cornelles. -We booked a quiet room. -Are all extras included in the bill? -We would like to change rooms, please. -Could we see a room? -How much does a double room including room service cost per week, please? -I would like a room with a telephone, please. -We booked two double rooms with a tv. -Can you give me the key to room number oh one nine? -Give me the key to the room, please. -There is a problem in the bill. -Is the phone bill included? -I am leaving. -Could you give me the key to room number five oh one? -We should leave today in the afternoon. -Would you ask for a taxi for me, please? -Could you send our luggage down to reception? -Can you send up my luggage to room number seven two eight, please? -Could you give us the key to the room? -We should leave on the fourteenth in the evening. -We want a room for nine nights, please. -Give us the keys to the room. -I want a room with a telephone, a tv and a shower, please. -I booked a double room. -Could you prepare the bill for me, please? -Send our suitcases to room number nine two six, please. -How much is a single room per week? -I want a room for a week, please. -I want to book a room, please. -I made a reservation. -Could you make out the bill, please? -Send up my luggage to room number oh three five. -The room is too cold. -Could you prepare our bill for room number three four one for us, please? -I think that there is a problem. -I want you to give us the keys to room number oh one oh, please. -I have made a reservation for a single room for Dulce Gumbau. -We booked a room. -Could you give us the key to room number six, please? -Could you make out the bill for us, please? -How much is a single room per day? -I made a reservation for Roberto Vázquez. -Do you have a room with a telephone and a tv available? -Send up our luggage to room number three four oh, please. -Could you send the luggage to the car? -What is the price for a double room with full board, please? -I want a double room for today. -We should leave today at twelve in the morning. -Please send my travel bag to my room. -Ask for a taxi. -How much is a single room? -Can you give us the keys to my room? -I want you to wake me up tomorrow at half past two, please. -Would you mind sending up my luggage to room number four nine oh, please? -Will you make out the bill for us, please? -Would you prepare the bill for me, please? -I have made a reservation. -I want a quiet, single room until tomorrow. -Could you send my suitcase to my room? -Would you mind waking me up at a quarter past three, please? -Can you give us the keys to room number nine two seven? -I have booked a single room with a tv. -I would like you to send my bag to room number eight oh three. -How much does a single room cost? -Is there a tv in the rooms? -I want you to send my luggage to room number seven three three, please. -I have a reservation. -Can you change my room? -Would you mind sending our suitcases to the taxi? -Can you wake us up at nine, please? -I have booked two rooms. -Can you give me the key to the room? -Good bye! -Can you give us the key to the room? -Can you wake me up at one? -Would you mind giving us the key to the room, please? -We have booked a quiet, single room. -I want you to give us the key to room number nine one one, please. -Can you give us the key to my room, please? -I am leaving. -We should leave this afternoon. -Would you mind putting my travel bags on the bus? -Could you give us the key to room number four seven one, please? -I want a quiet, single room until tomorrow, please. -Could we book two rooms? -I have made a reservation for a room with a telephone and a shower for Mr Borillo. -Could you ask for our taxi for us, please? -We have made a reservation. -Can you prepare the bill for room number one one one for me, please? -Would you mind sending our luggage to our room, please? -I want you to send my luggage to room number six one four, please. -Would you mind showing us a quiet room, please? -I have made a reservation for a room with a tv, a good view and a shower for Virginia Pérez. -I think that there is a mistake in the bill. -I am leaving on Thursday morning. -We booked a quiet, single room for thirty days for Lidia Llorens. -There is a lot of noise in the room. -We have booked two quiet rooms. -Would you mind showing us a room? -I want a room with a bathroom, a tv and a good view for Jorge Balaguer, please. -Does it have a tv? -Could we see a quiet, single room? -I think that there was a mistake in the bill. -It is very hot in the room. -I should leave tomorrow at a quarter past two in the afternoon. -Is there air conditioning in the rooms? -Can you wake us up at a quarter to one? -We booked a double room for twenty-nine days. -Could we book two quiet, single rooms until the twenty-eighth? -Can you call a taxi for me, please? -I want you to wake me up at half past seven. -Could you check the bill for me, please? -Do you have any quiet rooms available? -I think that there was a problem in the bill for room number nine six four. -Give me the keys to the room, please. -Could you call our taxi for room number two six eight for us, please? -Can you give us the key to the room? -Give me the key to room number four one three. -Could you send our luggage to reception? -We would like a room, please. -How much is a double room per week, please? -Could you wake us up tomorrow at half past nine, please? -We booked a single room for a night for Micaela Carpio. -Please ask for our taxi for room number one two eight. -How much does a double room with full board cost per day? -Wake me up at a quarter past one. -I have booked a room with a good view and a telephone. -Would you mind giving us the keys to the room, please? -I should leave today at a quarter past eight in the morning. -Can you ask for a taxi for room number oh one nine for us, please? -I think that there is an error. -We want a room. -I should leave on Thursday October the twenty-eighth at eight o'clock in the morning. -Could you wake me up at half past ten, please? -I would like you to wake me up at one, please. -We are leaving on the eleventh. -How many? -I have a reservation for Mr Borrás. -Can you send the suitcases to the bus? -There was a mistake in my bill. -Could I see a room, please? -Please make out the bill for room number nine one four. -Could you send up our luggage to room number four seven four? -Do you have any quiet rooms available? -I have booked a room until November the first for Mr Víctor Lobo. -Are extras included? -We have booked a room until next Sunday for Mr Ángel Viciano. -Could you give us the key to my room, please? -Can you make out the bill for room number two one two for me, please? -Could you send down my luggage to the room, please? -Could we book two single rooms for today? -Do you have a quiet, single room available? -Give me the key to room number three one oh. -We have booked a room for today. -There is too much noise in the room. -We should leave on the first in the morning. -I would like a quiet, single room, please. -We want a quiet, double room. -I want you to wake me up tomorrow at a quarter to twelve. -Could you wake us up at a quarter past one, please? -We are leaving on the fifteenth at a quarter to nine in the morning. -Are taxes included? -Could you call a taxi for me, please? -Could you send the travel bags down to reception? -Could you make out the bill for room number seven three two for us, please? -I have booked a room for eighteen days for Mr and Mrs Cabo. -Are there any rooms with a bath available? -Can you make out the bill for us, please? -I booked a quiet, double room with a tv for tonight for Miss Marina Miralles. -I think that there was a problem in the bill for room number seven eight six. -Do you have any rooms with a telephone available? -I am leaving today in the afternoon. -We should leave today at twelve in the morning. -We should leave on Monday evening. -I want a room with a tv, a shower and a telephone. -I booked a room for today for Mr Rafael Soler. -Could I book a double room? -We want a room until the twenty-eighth, please. -Do you have a quiet, single room available? -We would like to move to a room with a tv and a bathroom, please. -Do you have any quiet, single rooms available? -Can you change our room? -Are all taxes included? -Could you ask for a taxi for me, please? -I have booked a room for Mr Ortiz. -We are leaving on Sunday at nine o'clock in the morning. -Would you call a taxi for us, please? -Thank you very much! -Could you send down our suitcase to our room? -I have a reservation. -Can you wake me up tomorrow at a quarter past one? -Could you make out the bill for room number four one nine for us, please? -There is a mistake. -We have made a reservation. -Please send my travel bags to the room. -Do you have a quiet room available? -We have booked two rooms for a night. -Could you check my bill for me, please? -I have a reservation for a quiet room. -Could you give us the keys to room number five one five, please? -We want to book a room with air conditioning, please. -I have made a reservation for a room for Teresa Martínez. -Could you call my taxi for me, please? -I have booked a room with a shower, a tv and a telephone for Emilio Paches. -Could you give us the key to room number five nine, please? -Is everything included? -I have made a reservation for a room for Mr Ramos. -I think that there is a mistake in my bill for room number three three nine. -Is there air conditioning in the room? -We should leave on the fourteenth at a quarter past one in the afternoon. -I think that there is a mistake in the bill for room number seven one nine. -Wake us up at a quarter to one, please. -I think that there is a mistake in the bill. -Could you put the luggage in the taxi? -We want to book a double room for tonight. -Do you accept cash? -Could you put the luggage in the car? -I have booked a room for Sergio Orenga. -Could you call a taxi for room number one nine oh for me, please? -We would like you to give us the key to the room, please. -Is there a telephone in our room? -Can you explain the bill for us, please? -Can you give us the key to my room, please? -Is there a single room available? -We should leave. -Can you give me the key to room number two one four, please? -Would you mind changing my room? -Can you put our luggage in the car? -I want a quiet room. -Wake us up tomorrow at a quarter to six, please. -I want a room with a tv, a telephone and hot water until the twenty-seventh. -Please wake us up at one o'clock. -I have booked a quiet room for Tomás Fernández. -Can you give us the key to room number two five nine, please? -Please wake us up at a quarter past one. -I have booked a room. -Please give us the keys to room number six four one. -I have made a reservation for a room with a tv and a telephone for Miss Calleja. -I booked a room with a tv, a good view and a telephone. -I would like you to wake us up tomorrow at a quarter to six, please. -I want to change rooms. -We want to change rooms, please. -Do you have any double rooms available? -I would like a room. -We want a room. -I would like you to give me the key to our room, please. -I should leave on September the thirtieth. -Is there a room available? -Would you prepare the bill for me, please? -I would like a room with a tv and a telephone, please. -I have booked a room with a telephone. -Wake us up tomorrow at six o'clock, please. -We have booked a quiet room. -I have a reservation. -Could I see a quiet, single room, please? -How much is a single room, please? -Send my bags to the taxi. -We want to change rooms. -Can you send the bags to the station? -Would you mind showing me another room? -I want to book a room, please. -I have made a reservation. -What is the price for a single room for nine days, please? -We should leave tomorrow at one. -Please send my bags to room number oh two nine. -Could you send my bags to the bus? -We have booked a room for a night for Mr Federico Cornelles. -I made a reservation. -I think that there has been a problem. -Can you prepare the bill for room number five one one, please? -The room is very hot. -Could I see a quiet, single room? -Could you call a taxi for me, please? -Wake me up tomorrow at five. -Is room service included? -Could you give us the keys to the room, please? -I am leaving on Friday May the twenty-ninth at nine o'clock in the evening. -I want to book a quiet, single room for this week, please. -How much does a double room with full board cost? -Could you wake us up at half past seven? -We booked a single room. -Send our travel bags down to reception. -We have booked a room until February the second. -I would like to book a room for tonight, please. -Would you mind giving me the key to my room? -Would you mind waking me up at a quarter past eight, please? -Could you wake us up at half past six? -We want you to give us the key to the room. -I have a reservation for a quiet, single room for Enrique Díaz. -Do you have a room with hot water and air conditioning available? -Does the room have a telephone and air conditioning? -We should leave on the twenty-seventh at nine o'clock. -Would you call a taxi for me, please? -Would you mind waking us up tomorrow at a quarter to nine? -We booked a room with air conditioning until the day after tomorrow. -Can you ask for a taxi for us, please? -We have booked a double room. -I would like you to send my bags to my room, please. -I want a double room. -I leave tomorrow at twelve o'clock in the morning. -Are there air conditioning and a minibar in the rooms? -We booked two rooms. -Is there a safe in the rooms? -Do you have any quiet rooms available? -Can you give me the keys to room number one one three, please? -Please send up my suitcase to my room. -I have booked a double room with air conditioning, a telephone and a tv. -I want to book a room. -Could I see another room? -Do you have any double rooms available? -Are all taxes included? -Please wake us up tomorrow at two. -Could you give us the key to the room, please? -Put our suitcases in the car. -How much does a single room cost? -I am leaving on Friday April the eleventh at a quarter past eight in the evening. -I want you to give us the key to my room, please. -How much does a double room cost per week? -I made a reservation for a quiet, double room with a bathroom and a view of the town for Arturo Mira. -I have booked a room. -Is there a tv in the room, please? -I think that there has been a problem in my bill for room number six five. -How much does a single room cost, please? -Could you make out the bill for room number one four seven for me, please? -Is there hot water in the room, please? -Do you have a single room available? -How much does a single room cost for a day? -Can you call a taxi for room number three six for us, please? -How much is a double room per day? -We have made a reservation. -I have booked a quiet, double room for Paloma Montero. -We want to move to a room with a bathroom, please. -Could you wake me up at eleven o'clock, please? -Could you call a taxi for room number nine three six for us, please? -We are leaving on September the twenty-sixth at a quarter past seven. -I want you to give me the key to the room, please. -Could you ask for a taxi for me, please? -Would you accept cash? -I have a reservation for a room for Francisco Botella. -Could you prepare the bill for me, please? -I would like to book a quiet, single room with a telephone until the day after tomorrow, please. -Could you ask for a taxi for us, please? -Is there a tv in the rooms? -How much? -Okay! -Could you prepare the bill for room number two six three for us, please? -Is there a telephone in the rooms? -What time is it? -My name is Jesús González. -I have made a reservation for Teresa Ibá~nez. -I should leave today at eleven o'clock in the morning. -I would like you to send down my luggage to room number one two. -Where? -Could you ask for a taxi for room number six one four for us, please? -Would you mind waking me up tomorrow at half past four, please? -We would like to move to a quieter room, please. -I think that a mistake has been made in the bill. -Is the phone bill included? -A room for Virginia Álvarez, please. -We should leave today in the evening. -It is very hot in the room. -Could you prepare my bill for me, please? -Could I see a room, please? -We have booked a double room. -Would you accept credit cards? -I should leave on Thursday the sixteenth at half past nine in the evening. -Please send up my suitcase to my room. -I want a single room. -Are there any quiet, double rooms available? -We want you to send down our suitcases to the room. -Do you have a room available? -We are leaving on Saturday. -I have made a reservation for a room with a tv and a telephone for Inmaculada Colomer. -Send up my suitcases to room number two eight, please. -Would you mind waking us up at half past four, please? -There is an error in the bill for room number seven one nine. -Would you ask for our taxi for room number two three one for us, please? -Could you send up our travel bag to the room, please? -Would you mind sending our luggage to our room? -Can you move me to a warmer room? -We have booked a room. -Would you mind us taking a look at a room? -How many? -Give me the keys to our room. -Could you call a taxi for me, please? -I would like you to send my suitcase to room number seven oh three, please. -We have made a reservation. -Can you move me to a warmer room? -Good bye. -Please send my suitcase to the room. -Could you send my luggage to room number seven two three, please? -Is there a double room available? -Please give us the key to the room. -I think that there has been a mistake in the bill. -Wake us up tomorrow at one o'clock, please. -When? -My name is Emilio Navarro. -Would you mind sending up our bags to our room, please? -There is an error in our bill. -Is there a telephone in the room? -I have a reservation for a room with a telephone, a tv and a bath for Isabel Cabedo. -Could you wake me up tomorrow at twelve, please? -Could we see another room, please? -Could you make out the bill for me, please? -Could I book a room? -I should leave tomorrow at six. -We want you to give us the key to room number nine two one, please. -Send our bags to our room. -Can you move us to a quieter room? -I think that there is a mistake in my bill. -I should leave on September the eighth. -I have booked a double room for Mrs Inmaculada Colomer. -Could we book two rooms? -Send down our luggage to room number four one eight. -I should leave on the thirteenth at a quarter past four. -Do you have a single room available? -We booked two rooms. -I think that there is an error. -Would you mind moving us to a quieter room? -Give us the keys to room number nine nine two, please. -I want a room, please. -I have made a reservation for a room with a telephone, a view of the mountain, a bath and a tv for Ricardo Vilanova. -I want you to send up my luggage to my room. -I have made a reservation. -Could you wake us up tomorrow at four? -I think that there was an error in the bill. -I should leave tomorrow. -Would you mind giving us the key to the room? -I would like a room until next Tuesday. -Where? -Would you mind giving me the key to room number five two two, please? -I want to book a room. -Send my luggage to the station. -Could you carry the bags? -Could you give us the key to my room? -We would like a double room with a shower for a day. -Do you have a double room available? -There has been a mistake in the bill. -I would like to book a room for seven days. -We have booked a single room with a tv, a telephone and a bath until tomorrow for José Herrero. -Are extras included in the bill? -We have booked a double room. -The room is very cold. -I have booked a double room for today. -Good morning! -Can you send the travel bags to the car? -Could I see a quiet, double room with a tv, please? -Are all taxes included in the bill? -There is a mistake in the bill. -Would you mind putting my luggage on the bus? -I leave on March the twenty-seventh at half past two in the afternoon. -I booked a quiet, double room with a bath, a good view and a tv for Andrés Sanz. -Could you call a taxi for us, please? -Should we sign a registration form? -I have made a reservation for a room for Julia Vilanova. -Is there a quiet room available? -My name is Gregorio Pitarch. -Pardon? -Could you call my taxi for me, please? -I think that there is an error in the bill. -Do you have a room with a bath, air conditioning and a tv available? -Would you call a taxi for us, please? -Not at all. -We booked a room for Juan Rubio. -Could I book a room with air conditioning for this week? -We are leaving today at a quarter past ten in the evening. -It is too cold in the room. -I have a reservation. -We are leaving tomorrow at a quarter to three in the afternoon. -We want you to wake us up at eleven, please. -I would like you to give me the keys to room number six three eight. -Can you prepare the bill for us, please? -We booked a single room. -We want to book a room, please. -Would you mind sending my luggage to reception? -Could you call a taxi for me, please? -Does the room have air conditioning? -I should leave tomorrow at a quarter past ten in the evening. -I have booked a single room. -Do you have a single room available? -We should leave on the fourth. -Can I pay the bill in cash? -Could you give us the keys to our room? -I want a quiet, single room for today, please. -Should we sign the registration form? -We have made a reservation. -Would you mind showing me a room with a bath, please? -Wake me up tomorrow at a quarter to three, please. -I would like a room with a good view of the mountain and a shower, please. -I have booked a room. -Should we fill in the registration form? -Can you wake me up at two o'clock, please? -Can you wake us up at ten, please? diff --git a/samples/data/en_tok.txt b/samples/data/en_tok.txt deleted file mode 100644 index 8e2358597..000000000 --- a/samples/data/en_tok.txt +++ /dev/null @@ -1,1000 +0,0 @@ -Would you mind giving us the keys to the room , please ? -I have made a reservation for a quiet , double room with a telephone and a tv for Rosario Cabedo . -Would you mind moving me to a quieter room ? -I have booked a room . -I think that there is a problem . -Do you have any rooms with a tv , air conditioning and a safe available ? -Would you mind showing us a room with a tv ? -Does it have a telephone ? -I am leaving on the second at eight in the evening . -How much does a single room cost per week ? -Do you accept cash ? -My name is Victoria Piquer . -Would you mind sending up my luggage to room number three one oh ? -Do you accept cash ? -There is a mistake in my bill . -Could you explain the bill for me , please ? -I want to book a double room for a night . -Does it have a telephone ? -Could you wake us up at five o ' clock , please ? -I should leave on Tuesday March the seventh in the morning . -We have made a reservation . -Could you wake us up at twelve , please ? -Can you send our luggage to the taxi ? -I have a reservation for Julia Calatayud . -I have booked a room . -A room with a telephone and a view for Mrs Ramírez , please . -Can you send up our bags to the room , please ? -I have booked a room for Mr Roda . -How much does a double room with half board cost per day ? -Are all expenses included in the bill ? -I think that there is a mistake . -We want you to send up our suitcase to the room , please . -I would like a room . -Prepare the bill . -I want a quiet , double room , please . -Please make out the bill . -We want you to send our luggage to our room , please . -Could you wake us up at seven , please ? -Could we see a room ? -Could you wake us up tomorrow at a quarter to nine , please ? -Put my suitcases on the bus . -I want a room until the twentieth , please . -Could we book two rooms ? -Is there a quiet room available ? -I have booked a quiet room with a tv and a good view of the mountain for Rosario Cantero . -Do you have a double room available ? -I made a reservation . -Is the phone bill included ? -Can you give us the key to room number two nine four , please ? -Could you call a taxi , please ? -The room is very cold . -I have made a reservation . -Would you mind showing us another room ? -We would like you to send up our suitcases to our room . -Can you change our room ? -Will you call a taxi for me , please ? -The room is too hot . -Would you mind sending up my travel bags to the room ? -I would like to book a room with air conditioning for twenty days , please . -Could I see a quiet room , please ? -Wake me up at a quarter past two . -Could we see a room , please ? -I have booked a room until next Friday . -I am leaving on Friday . -Give me the key to room number one four nine , please . -I am leaving today at eight in the morning . -I have made a reservation for a room with a good view of the mountain and a bathroom for Pilar Jiménez . -Please check the bill . -Do you have a room with a tv , air conditioning and a minibar available ? -Could you wake me up tomorrow at two o ' clock ? -We have booked a double room until the day after tomorrow . -Could you call a taxi for me , please ? -I would like you to wake me up tomorrow at four , please . -I want a room for a night , please . -Could you ask for a taxi for us , please ? -I want to book a single room for twenty - five nights . -I have a reservation . -What is the date ? -Please make out the bill . -I have booked a single room . -I booked a room for Mr and Mrs Tena . -We have made a reservation . -Is everything included ? -Could you call a taxi for us , please ? -Carry our luggage . -Would you mind giving us the key to the room ? -I made a reservation for a double room for Miguel Ramírez . -We have booked a quiet room . -Can you give me the key to the room , please ? -Is the phone bill included ? -I made a reservation . -I want a room with air conditioning for today . -Would you mind changing my room ? -We have booked two rooms with a telephone . -I booked a double room with a telephone for today . -Is there air conditioning in my room , please ? -I have booked a room . -Would you mind sending the travel bags to the station ? -I have booked a quiet , double room for Enrique Velasco . -Could you send my bags down ? -Would you mind giving us the key to our room , please ? -We want a double room for a night , please . -I want to move to a room with a tv , please . -Do you have a quiet room available ? -I have booked a room with a telephone . -Please send my luggage to my room . -Is the phone bill included ? -Send my luggage to the station . -Would you call our taxi for us , please ? -How much does a single room with half board cost ? -The room is too cold . -Could you send the luggage to the bus ? -We made a reservation . -Could you make out our bill , please ? -I have booked a quiet , double room for Silvia Quereda . -My name is Jesús Rivera . -I have made a reservation for Sergio Moliner . -Do you have a room available ? -Could you ask for a taxi for me , please ? -Could you prepare our bill for room number six oh nine for us , please ? -Would you mind sending down our luggage to the room ? -Could you ask for a taxi for me , please ? -I would like to book a room until the twenty - fifth . -I want to book a double room with a telephone , please . -Could you wake me up at two ? -Can you wake us up at eleven , please ? -Do you have a room available ? -The room is too cold . -Could you call a taxi for me , please ? -How much is a single room with full board for a day ? -Could you wake me up at one o ' clock , please ? -I have a reservation for a quiet , double room with a tv and a view . -Could you send our suitcases to room number eight one nine , please ? -Sorry . -I have booked a quiet , double room with a tv and a telephone . -Could you wake me up at a quarter to eleven ? -I should leave this afternoon . -There has been a mistake in the bill . -We would like to change rooms , please . -I think that there was a mistake in the bill for room number eight seven four . -How much is a single room including breakfast , please ? -Could you wake me up at half past three , please ? -The room is very cold . -Would you mind giving me the key to the room ? -I have booked a room . -I would like to change rooms . -My name is Sergio Iborra . -I have made a reservation for Mr Miralles . -I have a reservation for Mr Ortiz . -We should leave on Saturday March the seventh . -Can you wake us up at half past ten , please ? -I would like you to send down our luggage to our room , please . -How much is a single room for a night ? -How much is a double room per day , please ? -I want a double room . -I want you to give me the key to room number three three three . -I booked a room . -Is breakfast included ? -The room is very expensive . -We are leaving on September the thirteenth at a quarter to eight in the morning . -Can you wake me up at half past eleven , please ? -Would you mind sending my bags to the bus ? -We are leaving tomorrow at four . -I think that there has been an error in the bill . -Would you mind me taking a look at a room ? -I want you to send down our luggage to the room , please . -Could you give me the keys to room number six one three , please ? -How much does a double room including room service cost , please ? -Can you explain the bill for room number oh one four for me , please ? -The room is all right . -I made a reservation . -I have made a reservation for a quiet , double room with a telephone and a shower . -We should leave tomorrow evening . -Are there a tv and air conditioning in the room ? -Wake me up tomorrow at half past ten , please . -Please give me the key to room number nine one oh . -We have booked a double room with a bath for today . -Does it have air conditioning , a telephone and a tv ? -A mistake has been made in our bill . -Could I see a single room with air conditioning ? -How much is a double room ? -Could I see a double room , please ? -Is there a quiet room available ? -I made a reservation for a room with a tv and a telephone for Roberto Arnau . -I have booked a room with a tv and a bath for Gerardo Ródenas . -I would like a room , please . -We have booked a room . -We are leaving on Thursday September the thirtieth at half past four . -Could I see a double room with a tv ? -Is everything included ? -Could you give us the key to the room , please ? -Can you give me the key to room number seven seven three ? -Would you make out the bill for us , please ? -We should leave on Thursday April the ninth in the afternoon . -I am leaving today at six in the morning . -I booked a quiet , double room . -I booked a quiet room for Amelia Peinado . -Would you ask for a taxi for room number nine eight one for me , please ? -How much does a double room cost , please ? -Would you ask for a taxi for us , please ? -Could you explain our bill for room number nine four one , please ? -Is everything included ? -Can you prepare my bill for me , please ? -I have made a reservation for a quiet , double room with a view , a shower and a telephone for Carmelo Peris . -Could you give me the key to the room , please ? -Could we see another room , please ? -Can you carry the travel bags ? -I would like to move to a quieter room , please . -Could you wake us up tomorrow at a quarter past nine ? -I want to book a room , please . -Could we book two double rooms for fifteen days ? -It does not matter ! -Could we book two rooms ? -Do you have a double room available ? -Please prepare the bill . -I would like a room , please . -Is room service included ? -We are leaving this evening . -Send up our luggage to our room . -My name is Federico Viciano . -I have booked a quiet room . -I have a reservation for Mrs Guijarro . -Would you mind waking us up at half past one ? -Do you have any quiet , single rooms available ? -We should leave tomorrow . -I have booked a room . -Could you make out the bill for me , please ? -Could I see a quiet , double room , please ? -A mistake has been made in the bill . -I have made a reservation for a room with a tv , a telephone and a view of the sea for Asunción Espinosa . -Does the room have air conditioning and hot water ? -I want to book a room until next Saturday . -Could you call a taxi for room number three one three for us , please ? -I want a room . -Can you wake me up at a quarter to one ? -A room , please . -I think that there was an error in the bill . -I would like a room for five nights . -We have booked a single room with a telephone for Mr and Mrs Santos . -Are all taxes included ? -Is the phone bill included ? -Please send our travel bag to room number five two seven . -I have booked a quiet room . -Ask for a taxi . -Can you give us the key to the room ? -Could you send the bags to the car ? -I have made a reservation for Susana Padilla . -Can you send down our luggage to the room , please ? -Would you mind sending down our luggage to the room , please ? -How much does a double room cost per week , please ? -Wake me up at half past one , please . -We should leave tomorrow at six . -I have made a reservation . -The room is too noisy . -I want you to give us the keys to room number seven one six , please . -Is everything included ? -I have booked a double room . -I would like to book a single room until the twenty - first , please . -Can you wake me up tomorrow at a quarter to three , please ? -Wake us up at one , please . -I should leave on Monday July the second in the afternoon . -Should I fill in a registration form , please ? -Could I see a room ? -I think that there was an error in the bill . -The room is very expensive . -We would like you to give us the keys to room number four three , please . -Could you explain the bill for us , please ? -Could you give us the keys to room number two two six , please ? -How much does a single room cost , please ? -Could we book two rooms for tonight ? -Could you wake us up at one , please ? -I think that there is a problem in my bill for room number nine three three . -No ! -My name is Marina Varela . -I want to change rooms , please . -The keys to room number eight two oh , please . -Are expenses included in the bill ? -I leave today . -Would you mind giving us the keys to the room ? -Do the rooms have air conditioning , please ? -Can you put the luggage in the car ? -Would you mind giving us the key to the room ? -Could you send down my suitcases to my room ? -I want a room with a telephone , a view , a tv and a bath for Marina Sáez . -I have a reservation for a quiet , double room with a view of the town and a bath for Ricardo Salsas . -Should we sign the registration form , please ? -Could you wake us up tomorrow at half past six ? -I should leave on Friday April the nineteenth at nine in the evening . -I think that a mistake has been made in the bill . -I would like to change rooms , please . -Is the phone bill included ? -Can you wake me up at a quarter to two , please ? -We are leaving on the seventh at half past eleven in the evening . -I would like you to wake us up at half past three , please . -We should leave tomorrow at nine . -Is there a minibar in the room ? -We booked a room until next Thursday . -Would you mind showing me a double room with a tv , please ? -I think that there is an error in the bill for room number eight three eight . -Could you make out the bill for room number nine oh one for us , please ? -I want to book a quiet , single room until November the first , please . -I have booked a quiet , single room with a view of the mountain and a tv for Mrs Gómez . -I have made a reservation . -Could I book a room ? -We are leaving on Monday April the nineteenth in the morning . -I have booked a quiet room . -Could you make out my bill for me , please ? -I should leave today at half past eleven in the morning . -Could you prepare the bill , please ? -Do you have a room available ? -I think that a mistake has been made in our bill for room number five one five . -I want to book a room , please . -Do you have a room available ? -Are extras included in the bill ? -I have booked a room . -We have made a reservation . -Can you ask for my taxi for room number seven one six for me , please ? -Can you wake me up at a quarter past four ? -Are all taxes included in the bill ? -Is breakfast included ? -I booked a room for Mr Gerardo Paches . -I have booked a single room for Mr Fernández . -I leave on the eighteenth at a quarter to seven in the morning . -Could you give us the key to room number six four seven , please ? -Could we see a room , please ? -Can you send our luggage to the bus ? -I leave on Friday the first at three . -Can you send down my luggage to the room ? -How much is a double room per day , please ? -There is a lot of noise in the room . -Is the phone bill included ? -The room is all right . -Is there hot water in the rooms , please ? -Would you mind changing my room ? -Could you put my travel bags in the taxi ? -We would like you to wake us up at a quarter to eight , please . -It is very hot in the room . -We want to change rooms . -Put my bags on the bus . -Does my room have air conditioning ? -Wake me up at six . -I am leaving tomorrow at a quarter to three in the afternoon . -Are there any rooms available ? -Can you move us to a room with a telephone and a tv ? -It does not matter . -Are there a telephone and a tv in the room ? -I should leave today . -We are leaving this morning . -Would you mind showing us a room with a telephone and a tv , please ? -Carry the luggage . -Is there air conditioning in the rooms ? -Could you send up my luggage to my room ? -I do not agree . -I have booked a single room . -I want to book a room for a day , please . -Could you give us the key to my room ? -Can you give me the key to the room , please ? -I have made a reservation . -I should leave tomorrow morning . -Should I fill in the registration form , please ? -Would you mind changing my room ? -Can you give me the keys to room number oh three oh , please ? -Can you send down our travel bags to room number four two three , please ? -I want to book a quiet , single room for a week , please . -Please explain the bill . -Could you ask for a taxi for room number seven two nine for us , please ? -I have booked a room for tonight for Manuela Valls . -I would like you to give us the key to room number one one , please . -Can you send down our luggage to room number four one four , please ? -I would like to change rooms , please . -We have booked a quiet , single room . -Please send up my travel bag to room number six three one . -Can you ask for a taxi , please ? -Can you ask for a taxi , please ? -Could you make out the bill for room number six two six , please ? -Give us the key to room number eight one four . -I want you to send up my luggage to the room , please . -Give me the key to room number six one eight , please . -Could you give us the key to room number six four oh ? -Could we see a quiet room , please ? -Yes ! -Send our luggage to room number eight two six , please . -Could you send down our luggage to room number three six two ? -I have booked a quiet room until tomorrow for Gregorio Cabedo . -We are leaving today at half past six in the afternoon . -Would you mind sending our travel bags to room number two three five , please ? -We made a reservation . -Could you explain the bill for me , please ? -I would like to move to a room with a minibar and air conditioning , please . -We would like you to wake us up at a quarter past eleven . -We would like you to wake us up at nine o ' clock . -I booked a room . -Can you move us to a room with a tv ? -I should leave tomorrow at ten o ' clock in the morning . -Can you explain the bill , please ? -Please send up my luggage to my room . -Can you send our luggage to the bus ? -Should we sign the registration form ? -I would like you to wake me up tomorrow at ten , please . -We booked a double room . -We are leaving tomorrow at five o ' clock in the afternoon . -Is there air conditioning in the room ? -Would you mind giving me the key to room number three three three , please ? -I am leaving tomorrow at half past four in the afternoon . -We should leave on Monday April the fourth at half past twelve in the morning . -What is the price for a double room with half board per day , please ? -It is too hot in the room . -How much does a double room cost ? -Would you mind showing us a room with a telephone , please ? -How much does a single room cost per day ? -I should leave . -Could you carry my bags ? -How much does a double room cost for eighteen days , please ? -Can you wake us up tomorrow at five o ' clock ? -Are all expenses included in the bill ? -I have made a reservation for a double room with a bath and a telephone for Paloma Betoret . -I would like you to give me the keys to room number two three nine . -Could you check the bill for me , please ? -I want you to send up my luggage to my room , please . -Can you give us the key to room number one four oh , please ? -Would you mind giving me the keys to the room , please ? -Could you wake us up at a quarter to six ? -How much is a single room , please ? -Could we book two quiet rooms for tonight ? -I have booked a quiet , single room . -The room is very expensive . -I think that there was a problem in the bill . -I want a double room for today . -I want a single room . -I booked a room for Alicia Morales . -Okay . -Please give me the key to our room . -We want a room , please . -I would like to book two double rooms , please . -I am leaving tomorrow at a quarter to ten in the evening . -I want to change rooms . -The room is very hot . -I made a reservation for a quiet , single room with a good view of the sea , a telephone and a tv . -Would you mind changing our room ? -We have booked a single room until the seventeenth . -I have a reservation . -Do you accept traveler checks ? -We have booked a room . -How much ? -How much is a single room per day , please ? -Do you have a quiet , double room available ? -Will you prepare the bill for us , please ? -Can you wake us up at four , please ? -We have booked a double room . -The room is all right . -Could you check the bill , please ? -Please call my taxi . -I would like you to give us the key to my room . -I want you to wake us up at half past nine . -I would like a quiet , double room with a telephone , a good view of the mountain , a bath and a tv for Mrs Berrueco , please . -Could you ask for my taxi for me , please ? -Could you ask for a taxi for me , please ? -I think that there has been an error in the bill . -The room is too cold . -Could you call my taxi for me , please ? -How much does a single room cost per week ? -Would you mind me taking a look at a double room ? -The room is all right . -Send our luggage to our room . -Could you carry my suitcases ? -I want a room for twenty - nine days . -Should I fill in the registration form ? -I want a room . -We want a single room until next Friday . -Give us the key to room number seven oh five . -Could you ask for a taxi , please ? -We should leave on Saturday May the fourteenth . -Prepare our bill . -I should leave tomorrow at a quarter to five in the afternoon . -I want a room . -There is a mistake in the bill . -Do you have a single room available ? -Is there a room available ? -We have booked a single room for Luis Llopis . -We want a double room with a shower and air conditioning . -We have made a reservation . -Is the phone bill included ? -Please give me the key to the room . -I think that a mistake has been made in our bill . -I have a reservation for a quiet , double room with a tv , a bathroom and a good view of the sea . -Would you mind waking me up at half past ten , please ? -Please make out the bill . -I have booked a room for Jesús Barberá . -I want a room for a day , please . -Could you call a taxi for me , please ? -I want you to send my luggage to my room , please . -Is there a tv in my room ? -I booked a quiet room for twenty - six nights for Celestino Puig . -I leave on September the sixteenth at a quarter to twelve in the morning . -How much does a single room including breakfast cost per week ? -Please send up our travel bag to room number one seven four . -Please prepare the bill . -I should leave on the third at a quarter past ten in the morning . -Do you have a room available ? -I have booked a quiet room with a bathroom , a telephone and a tv . -Wake us up at a quarter past six . -Could we see a double room ? -Could you wake us up at half past six ? -I think that a mistake has been made in the bill . -I would like you to give me the key to the room , please . -Please ask for a taxi . -Do you have any rooms available ? -We have booked a room for ten weeks for Rosario Cornelles . -We booked a quiet room . -Are all extras included in the bill ? -We would like to change rooms , please . -Could we see a room ? -How much does a double room including room service cost per week , please ? -I would like a room with a telephone , please . -We booked two double rooms with a tv . -Can you give me the key to room number oh one nine ? -Give me the key to the room , please . -There is a problem in the bill . -Is the phone bill included ? -I am leaving . -Could you give me the key to room number five oh one ? -We should leave today in the afternoon . -Would you ask for a taxi for me , please ? -Could you send our luggage down to reception ? -Can you send up my luggage to room number seven two eight , please ? -Could you give us the key to the room ? -We should leave on the fourteenth in the evening . -We want a room for nine nights , please . -Give us the keys to the room . -I want a room with a telephone , a tv and a shower , please . -I booked a double room . -Could you prepare the bill for me , please ? -Send our suitcases to room number nine two six , please . -How much is a single room per week ? -I want a room for a week , please . -I want to book a room , please . -I made a reservation . -Could you make out the bill , please ? -Send up my luggage to room number oh three five . -The room is too cold . -Could you prepare our bill for room number three four one for us , please ? -I think that there is a problem . -I want you to give us the keys to room number oh one oh , please . -I have made a reservation for a single room for Dulce Gumbau . -We booked a room . -Could you give us the key to room number six , please ? -Could you make out the bill for us , please ? -How much is a single room per day ? -I made a reservation for Roberto Vázquez . -Do you have a room with a telephone and a tv available ? -Send up our luggage to room number three four oh , please . -Could you send the luggage to the car ? -What is the price for a double room with full board , please ? -I want a double room for today . -We should leave today at twelve in the morning . -Please send my travel bag to my room . -Ask for a taxi . -How much is a single room ? -Can you give us the keys to my room ? -I want you to wake me up tomorrow at half past two , please . -Would you mind sending up my luggage to room number four nine oh , please ? -Will you make out the bill for us , please ? -Would you prepare the bill for me , please ? -I have made a reservation . -I want a quiet , single room until tomorrow . -Could you send my suitcase to my room ? -Would you mind waking me up at a quarter past three , please ? -Can you give us the keys to room number nine two seven ? -I have booked a single room with a tv . -I would like you to send my bag to room number eight oh three . -How much does a single room cost ? -Is there a tv in the rooms ? -I want you to send my luggage to room number seven three three , please . -I have a reservation . -Can you change my room ? -Would you mind sending our suitcases to the taxi ? -Can you wake us up at nine , please ? -I have booked two rooms . -Can you give me the key to the room ? -Good bye ! -Can you give us the key to the room ? -Can you wake me up at one ? -Would you mind giving us the key to the room , please ? -We have booked a quiet , single room . -I want you to give us the key to room number nine one one , please . -Can you give us the key to my room , please ? -I am leaving . -We should leave this afternoon . -Would you mind putting my travel bags on the bus ? -Could you give us the key to room number four seven one , please ? -I want a quiet , single room until tomorrow , please . -Could we book two rooms ? -I have made a reservation for a room with a telephone and a shower for Mr Borillo . -Could you ask for our taxi for us , please ? -We have made a reservation . -Can you prepare the bill for room number one one one for me , please ? -Would you mind sending our luggage to our room , please ? -I want you to send my luggage to room number six one four , please . -Would you mind showing us a quiet room , please ? -I have made a reservation for a room with a tv , a good view and a shower for Virginia Pérez . -I think that there is a mistake in the bill . -I am leaving on Thursday morning . -We booked a quiet , single room for thirty days for Lidia Llorens . -There is a lot of noise in the room . -We have booked two quiet rooms . -Would you mind showing us a room ? -I want a room with a bathroom , a tv and a good view for Jorge Balaguer , please . -Does it have a tv ? -Could we see a quiet , single room ? -I think that there was a mistake in the bill . -It is very hot in the room . -I should leave tomorrow at a quarter past two in the afternoon . -Is there air conditioning in the rooms ? -Can you wake us up at a quarter to one ? -We booked a double room for twenty - nine days . -Could we book two quiet , single rooms until the twenty - eighth ? -Can you call a taxi for me , please ? -I want you to wake me up at half past seven . -Could you check the bill for me , please ? -Do you have any quiet rooms available ? -I think that there was a problem in the bill for room number nine six four . -Give me the keys to the room , please . -Could you call our taxi for room number two six eight for us , please ? -Can you give us the key to the room ? -Give me the key to room number four one three . -Could you send our luggage to reception ? -We would like a room , please . -How much is a double room per week , please ? -Could you wake us up tomorrow at half past nine , please ? -We booked a single room for a night for Micaela Carpio . -Please ask for our taxi for room number one two eight . -How much does a double room with full board cost per day ? -Wake me up at a quarter past one . -I have booked a room with a good view and a telephone . -Would you mind giving us the keys to the room , please ? -I should leave today at a quarter past eight in the morning . -Can you ask for a taxi for room number oh one nine for us , please ? -I think that there is an error . -We want a room . -I should leave on Thursday October the twenty - eighth at eight o ' clock in the morning . -Could you wake me up at half past ten , please ? -I would like you to wake me up at one , please . -We are leaving on the eleventh . -How many ? -I have a reservation for Mr Borrás . -Can you send the suitcases to the bus ? -There was a mistake in my bill . -Could I see a room , please ? -Please make out the bill for room number nine one four . -Could you send up our luggage to room number four seven four ? -Do you have any quiet rooms available ? -I have booked a room until November the first for Mr Víctor Lobo . -Are extras included ? -We have booked a room until next Sunday for Mr Ángel Viciano . -Could you give us the key to my room , please ? -Can you make out the bill for room number two one two for me , please ? -Could you send down my luggage to the room , please ? -Could we book two single rooms for today ? -Do you have a quiet , single room available ? -Give me the key to room number three one oh . -We have booked a room for today . -There is too much noise in the room . -We should leave on the first in the morning . -I would like a quiet , single room , please . -We want a quiet , double room . -I want you to wake me up tomorrow at a quarter to twelve . -Could you wake us up at a quarter past one , please ? -We are leaving on the fifteenth at a quarter to nine in the morning . -Are taxes included ? -Could you call a taxi for me , please ? -Could you send the travel bags down to reception ? -Could you make out the bill for room number seven three two for us , please ? -I have booked a room for eighteen days for Mr and Mrs Cabo . -Are there any rooms with a bath available ? -Can you make out the bill for us , please ? -I booked a quiet , double room with a tv for tonight for Miss Marina Miralles . -I think that there was a problem in the bill for room number seven eight six . -Do you have any rooms with a telephone available ? -I am leaving today in the afternoon . -We should leave today at twelve in the morning . -We should leave on Monday evening . -I want a room with a tv , a shower and a telephone . -I booked a room for today for Mr Rafael Soler . -Could I book a double room ? -We want a room until the twenty - eighth , please . -Do you have a quiet , single room available ? -We would like to move to a room with a tv and a bathroom , please . -Do you have any quiet , single rooms available ? -Can you change our room ? -Are all taxes included ? -Could you ask for a taxi for me , please ? -I have booked a room for Mr Ortiz . -We are leaving on Sunday at nine o ' clock in the morning . -Would you call a taxi for us , please ? -Thank you very much ! -Could you send down our suitcase to our room ? -I have a reservation . -Can you wake me up tomorrow at a quarter past one ? -Could you make out the bill for room number four one nine for us , please ? -There is a mistake . -We have made a reservation . -Please send my travel bags to the room . -Do you have a quiet room available ? -We have booked two rooms for a night . -Could you check my bill for me , please ? -I have a reservation for a quiet room . -Could you give us the keys to room number five one five , please ? -We want to book a room with air conditioning , please . -I have made a reservation for a room for Teresa Martínez . -Could you call my taxi for me , please ? -I have booked a room with a shower , a tv and a telephone for Emilio Paches . -Could you give us the key to room number five nine , please ? -Is everything included ? -I have made a reservation for a room for Mr Ramos . -I think that there is a mistake in my bill for room number three three nine . -Is there air conditioning in the room ? -We should leave on the fourteenth at a quarter past one in the afternoon . -I think that there is a mistake in the bill for room number seven one nine . -Wake us up at a quarter to one , please . -I think that there is a mistake in the bill . -Could you put the luggage in the taxi ? -We want to book a double room for tonight . -Do you accept cash ? -Could you put the luggage in the car ? -I have booked a room for Sergio Orenga . -Could you call a taxi for room number one nine oh for me , please ? -We would like you to give us the key to the room , please . -Is there a telephone in our room ? -Can you explain the bill for us , please ? -Can you give us the key to my room , please ? -Is there a single room available ? -We should leave . -Can you give me the key to room number two one four , please ? -Would you mind changing my room ? -Can you put our luggage in the car ? -I want a quiet room . -Wake us up tomorrow at a quarter to six , please . -I want a room with a tv , a telephone and hot water until the twenty - seventh . -Please wake us up at one o ' clock . -I have booked a quiet room for Tomás Fernández . -Can you give us the key to room number two five nine , please ? -Please wake us up at a quarter past one . -I have booked a room . -Please give us the keys to room number six four one . -I have made a reservation for a room with a tv and a telephone for Miss Calleja . -I booked a room with a tv , a good view and a telephone . -I would like you to wake us up tomorrow at a quarter to six , please . -I want to change rooms . -We want to change rooms , please . -Do you have any double rooms available ? -I would like a room . -We want a room . -I would like you to give me the key to our room , please . -I should leave on September the thirtieth . -Is there a room available ? -Would you prepare the bill for me , please ? -I would like a room with a tv and a telephone , please . -I have booked a room with a telephone . -Wake us up tomorrow at six o ' clock , please . -We have booked a quiet room . -I have a reservation . -Could I see a quiet , single room , please ? -How much is a single room , please ? -Send my bags to the taxi . -We want to change rooms . -Can you send the bags to the station ? -Would you mind showing me another room ? -I want to book a room , please . -I have made a reservation . -What is the price for a single room for nine days , please ? -We should leave tomorrow at one . -Please send my bags to room number oh two nine . -Could you send my bags to the bus ? -We have booked a room for a night for Mr Federico Cornelles . -I made a reservation . -I think that there has been a problem . -Can you prepare the bill for room number five one one , please ? -The room is very hot . -Could I see a quiet , single room ? -Could you call a taxi for me , please ? -Wake me up tomorrow at five . -Is room service included ? -Could you give us the keys to the room , please ? -I am leaving on Friday May the twenty - ninth at nine o ' clock in the evening . -I want to book a quiet , single room for this week , please . -How much does a double room with full board cost ? -Could you wake us up at half past seven ? -We booked a single room . -Send our travel bags down to reception . -We have booked a room until February the second . -I would like to book a room for tonight , please . -Would you mind giving me the key to my room ? -Would you mind waking me up at a quarter past eight , please ? -Could you wake us up at half past six ? -We want you to give us the key to the room . -I have a reservation for a quiet , single room for Enrique Díaz . -Do you have a room with hot water and air conditioning available ? -Does the room have a telephone and air conditioning ? -We should leave on the twenty - seventh at nine o ' clock . -Would you call a taxi for me , please ? -Would you mind waking us up tomorrow at a quarter to nine ? -We booked a room with air conditioning until the day after tomorrow . -Can you ask for a taxi for us , please ? -We have booked a double room . -I would like you to send my bags to my room , please . -I want a double room . -I leave tomorrow at twelve o ' clock in the morning . -Are there air conditioning and a minibar in the rooms ? -We booked two rooms . -Is there a safe in the rooms ? -Do you have any quiet rooms available ? -Can you give me the keys to room number one one three , please ? -Please send up my suitcase to my room . -I have booked a double room with air conditioning , a telephone and a tv . -I want to book a room . -Could I see another room ? -Do you have any double rooms available ? -Are all taxes included ? -Please wake us up tomorrow at two . -Could you give us the key to the room , please ? -Put our suitcases in the car . -How much does a single room cost ? -I am leaving on Friday April the eleventh at a quarter past eight in the evening . -I want you to give us the key to my room , please . -How much does a double room cost per week ? -I made a reservation for a quiet , double room with a bathroom and a view of the town for Arturo Mira . -I have booked a room . -Is there a tv in the room , please ? -I think that there has been a problem in my bill for room number six five . -How much does a single room cost , please ? -Could you make out the bill for room number one four seven for me , please ? -Is there hot water in the room , please ? -Do you have a single room available ? -How much does a single room cost for a day ? -Can you call a taxi for room number three six for us , please ? -How much is a double room per day ? -We have made a reservation . -I have booked a quiet , double room for Paloma Montero . -We want to move to a room with a bathroom , please . -Could you wake me up at eleven o ' clock , please ? -Could you call a taxi for room number nine three six for us , please ? -We are leaving on September the twenty - sixth at a quarter past seven . -I want you to give me the key to the room , please . -Could you ask for a taxi for me , please ? -Would you accept cash ? -I have a reservation for a room for Francisco Botella . -Could you prepare the bill for me , please ? -I would like to book a quiet , single room with a telephone until the day after tomorrow , please . -Could you ask for a taxi for us , please ? -Is there a tv in the rooms ? -How much ? -Okay ! -Could you prepare the bill for room number two six three for us , please ? -Is there a telephone in the rooms ? -What time is it ? -My name is Jesús González . -I have made a reservation for Teresa Ibá ~ nez . -I should leave today at eleven o ' clock in the morning . -I would like you to send down my luggage to room number one two . -Where ? -Could you ask for a taxi for room number six one four for us , please ? -Would you mind waking me up tomorrow at half past four , please ? -We would like to move to a quieter room , please . -I think that a mistake has been made in the bill . -Is the phone bill included ? -A room for Virginia Álvarez , please . -We should leave today in the evening . -It is very hot in the room . -Could you prepare my bill for me , please ? -Could I see a room , please ? -We have booked a double room . -Would you accept credit cards ? -I should leave on Thursday the sixteenth at half past nine in the evening . -Please send up my suitcase to my room . -I want a single room . -Are there any quiet , double rooms available ? -We want you to send down our suitcases to the room . -Do you have a room available ? -We are leaving on Saturday . -I have made a reservation for a room with a tv and a telephone for Inmaculada Colomer . -Send up my suitcases to room number two eight , please . -Would you mind waking us up at half past four , please ? -There is an error in the bill for room number seven one nine . -Would you ask for our taxi for room number two three one for us , please ? -Could you send up our travel bag to the room , please ? -Would you mind sending our luggage to our room ? -Can you move me to a warmer room ? -We have booked a room . -Would you mind us taking a look at a room ? -How many ? -Give me the keys to our room . -Could you call a taxi for me , please ? -I would like you to send my suitcase to room number seven oh three , please . -We have made a reservation . -Can you move me to a warmer room ? -Good bye . -Please send my suitcase to the room . -Could you send my luggage to room number seven two three , please ? -Is there a double room available ? -Please give us the key to the room . -I think that there has been a mistake in the bill . -Wake us up tomorrow at one o ' clock , please . -When ? -My name is Emilio Navarro . -Would you mind sending up our bags to our room , please ? -There is an error in our bill . -Is there a telephone in the room ? -I have a reservation for a room with a telephone , a tv and a bath for Isabel Cabedo . -Could you wake me up tomorrow at twelve , please ? -Could we see another room , please ? -Could you make out the bill for me , please ? -Could I book a room ? -I should leave tomorrow at six . -We want you to give us the key to room number nine two one , please . -Send our bags to our room . -Can you move us to a quieter room ? -I think that there is a mistake in my bill . -I should leave on September the eighth . -I have booked a double room for Mrs Inmaculada Colomer . -Could we book two rooms ? -Send down our luggage to room number four one eight . -I should leave on the thirteenth at a quarter past four . -Do you have a single room available ? -We booked two rooms . -I think that there is an error . -Would you mind moving us to a quieter room ? -Give us the keys to room number nine nine two , please . -I want a room , please . -I have made a reservation for a room with a telephone , a view of the mountain , a bath and a tv for Ricardo Vilanova . -I want you to send up my luggage to my room . -I have made a reservation . -Could you wake us up tomorrow at four ? -I think that there was an error in the bill . -I should leave tomorrow . -Would you mind giving us the key to the room ? -I would like a room until next Tuesday . -Where ? -Would you mind giving me the key to room number five two two , please ? -I want to book a room . -Send my luggage to the station . -Could you carry the bags ? -Could you give us the key to my room ? -We would like a double room with a shower for a day . -Do you have a double room available ? -There has been a mistake in the bill . -I would like to book a room for seven days . -We have booked a single room with a tv , a telephone and a bath until tomorrow for José Herrero . -Are extras included in the bill ? -We have booked a double room . -The room is very cold . -I have booked a double room for today . -Good morning ! -Can you send the travel bags to the car ? -Could I see a quiet , double room with a tv , please ? -Are all taxes included in the bill ? -There is a mistake in the bill . -Would you mind putting my luggage on the bus ? -I leave on March the twenty - seventh at half past two in the afternoon . -I booked a quiet , double room with a bath , a good view and a tv for Andrés Sanz . -Could you call a taxi for us , please ? -Should we sign a registration form ? -I have made a reservation for a room for Julia Vilanova . -Is there a quiet room available ? -My name is Gregorio Pitarch . -Pardon ? -Could you call my taxi for me , please ? -I think that there is an error in the bill . -Do you have a room with a bath , air conditioning and a tv available ? -Would you call a taxi for us , please ? -Not at all . -We booked a room for Juan Rubio . -Could I book a room with air conditioning for this week ? -We are leaving today at a quarter past ten in the evening . -It is too cold in the room . -I have a reservation . -We are leaving tomorrow at a quarter to three in the afternoon . -We want you to wake us up at eleven , please . -I would like you to give me the keys to room number six three eight . -Can you prepare the bill for us , please ? -We booked a single room . -We want to book a room , please . -Would you mind sending my luggage to reception ? -Could you call a taxi for me , please ? -Does the room have air conditioning ? -I should leave tomorrow at a quarter past ten in the evening . -I have booked a single room . -Do you have a single room available ? -We should leave on the fourth . -Can I pay the bill in cash ? -Could you give us the keys to our room ? -I want a quiet , single room for today , please . -Should we sign the registration form ? -We have made a reservation . -Would you mind showing me a room with a bath , please ? -Wake me up tomorrow at a quarter to three , please . -I would like a room with a good view of the mountain and a shower , please . -I have booked a room . -Should we fill in the registration form ? -Can you wake me up at two o ' clock , please ? -Can you wake us up at ten , please ? diff --git a/samples/data/smt.cfg b/samples/data/smt.cfg deleted file mode 100644 index b438a898f..000000000 --- a/samples/data/smt.cfg +++ /dev/null @@ -1,29 +0,0 @@ -# Translation model prefix --tm tm/src_trg - -# Language model --lm lm/trg.lm - -# W parameter (maximum number of translation options to be considered per each source phrase) --W 10 - -# S parameter (maximum number of hypotheses that can be stored in each stack) --S 10 - -# A parameter (Maximum length in words of the source phrases to be translated) --A 7 - -# Degree of non-monotonicity --nomon 0 - -# Heuristic function used --h 6 - -# Best-first search flag --be - -# Translation model weights --tmw 0 0.5 1 1 1 1 0 1 - -# Set online learning parameters (ol_alg, lr_policy, l_stepsize, em_iters, e_par, r_par) --olp 0 0 1 5 1 0 diff --git a/samples/data/sp-hc.xml b/samples/data/sp-hc.xml deleted file mode 100644 index c1e448e2a..000000000 --- a/samples/data/sp-hc.xml +++ /dev/null @@ -1,282 +0,0 @@ - - - - - Spanish - - - Noun - - - Verb - - - Article - - - - Spanish Orthography - - - - a - - - - - e - - - - - i - - - - - o - - - - - u - - - - - á - - - - - é - - - - - í - - - - - ó - - - - - y - - - - - - b - - - - - c - - - - - d - - - - - f - - - - - g - - - - - h - - - - - j - - - - - k - - - - - l - - - - - m - - - - - n - - - - - p - - - - - q - - - - - r - - - - - s - - - - - t - - - - - v - - - - - x - - - - - z - - - - - - - + - - - - - - - Any - - - - - morphophonemic - - - Preterite 3SG - - - - - - - - - - - - - +ó - - - - - rule1 - PAST - - - Plural - - - - - - - - - - - - - +s - - - - - rule2 - PLURAL - - - - - - - dios - - - le1 - god - - - - - cre - - - le2 - create - - - - - mundo - - - le3 - world - - - - - el - - - le4 - the - - - - - cuenta - - - le5 - bill - - - - - surface - - - - \ No newline at end of file diff --git a/samples/data/sp.txt b/samples/data/sp.txt deleted file mode 100644 index f50ee4410..000000000 --- a/samples/data/sp.txt +++ /dev/null @@ -1,1000 +0,0 @@ -¿Le importaría darnos las llaves de la habitación, por favor? -He hecho la reserva de una habitación tranquila doble con teléfono y televisión a nombre de Rosario Cabedo. -¿Le importaría cambiarme a otra habitación más tranquila? -Por favor, tengo reservada una habitación. -Me parece que existe un problema. -¿Tiene habitaciones libres con televisión, aire acondicionado y caja fuerte? -¿Le importaría mostrarnos una habitación con televisión? -¿Tiene teléfono? -Voy a marcharme el dos a las ocho de la noche. -¿Cuánto cuesta una habitación individual por semana? -¿Acepta dinero en efectivo? -Mi nombre es Victoria Piquer. -¿Les importaría subirme los bultos a la habitación tres diez? -¿Aceptan dinero en efectivo? -Hay una equivocación en mi cuenta. -Por favor, me detalla la cuenta. -Deseo reservar una habitación doble para una noche. -¿Tiene teléfono? -¿Nos podrían despertar a las cinco en punto, por favor? -Me tengo que ir el martes siete de marzo por la mañana. -Hemos hecho una reserva, por favor. -Por favor, ¿nos podrían despertar a las doce? -¿Nos pueden llevar nuestro equipaje al taxi? -Tengo una reserva a nombre de Julia Calatayud. -Por favor, tengo reservada una habitación. -Por favor, una habitación con teléfono y vistas a nombre de la señora Ramírez. -Por favor, ¿pueden subirnos ustedes las bolsas a la habitación? -He reservado una habitación a nombre del señor Roda. -¿Cuánto cuesta una habitación doble por día con media pensión? -¿Están apuntados en la cuenta todos los gastos? -Pienso que existe una equivocación. -Deseamos que suba nuestra maleta a la habitación, por favor. -Desearía una habitación. -Prepare la cuenta. -Quisiera una habitación doble y tranquila, por favor. -Por favor, haga la cuenta. -Queremos que nos lleven nuestro equipaje a nuestra habitación, por favor. -¿Nos podrían despertar a las siete, por favor? -¿Podríamos ver alguna habitación? -¿Podría despertarnos mañana a las nueve menos cuarto, por favor? -Súbame las maletas al autobús. -Quisiera una habitación hasta el día veinte, por favor. -¿Podríamos reservar dos habitaciones? -¿Hay alguna habitación tranquila libre? -Tengo reservada una habitación tranquila con televisión y una buena vista de la montaña a nombre de Rosario Cantero. -¿Tiene libre una habitación doble? -Hice una reserva, por favor. -¿Está apuntado el recibo del teléfono? -¿Puede darnos usted la llave de la habitación número dos nueve cuatro, por favor? -¿Podría llamar a un taxi? -La habitación es muy fría. -He hecho una reserva. -¿Le importaría mostrarnos otra habitación? -Quisiéramos que subiese nuestras maletas a nuestra habitación. -¿Podría cambiarnos de habitación? -¿Me quieren llamar a un taxi? -La habitación es demasiado calurosa. -¿Le importaría subirme las bolsas de viaje a la habitación? -Desearía reservar una habitación con aire acondicionado para veinte días, por favor. -Por favor, ¿podría ver alguna habitación tranquila? -Despiérteme a las dos y cuarto. -¿Podríamos ver una habitación, por favor? -Tengo reservada una habitación hasta el próximo viernes, por favor. -Voy a marcharme el viernes. -Déme la llave de la habitación número uno cuatro nueve, por favor. -Me voy a ir hoy mismo a las ocho de la mañana. -Tengo hecha la reserva de una habitación con buena vista de la montaña y cuarto de baño a nombre de Pilar Jiménez. -Por favor, repase la factura. -¿Tiene alguna habitación libre con televisión, aire acondicionado y servicio de bar? -¿Me podrían despertar mañana a las dos en punto? -Hemos reservado una habitación doble hasta pasado mañana. -Por favor, ¿me podría llamar a un taxi? -Por favor, quisiera que me despertara mañana a las cuatro. -Quiero una habitación para una noche, por favor. -¿Podría pedirnos un taxi?, por favor. -Deseo reservar una habitación individual para veinticinco noches. -Tengo una reserva. -¿A qué día estamos? -Por favor, haga la cuenta. -Tengo reservada una habitación individual. -Por favor, reservé una habitación a nombre del señor y la señora Tena. -Hemos hecho una reserva, por favor. -¿Está todo? -Por favor, nos llama a un taxi. -Lleve nuestro equipaje. -¿Les importaría darnos la llave de la habitación? -Hice la reserva de una habitación doble a nombre de Miguel Ramírez. -Por favor, hemos reservado una habitación tranquila. -Por favor, ¿me puede dar la llave de la habitación? -¿Está apuntado el recibo del teléfono? -Por favor, hice una reserva. -Quiero una habitación con aire acondicionado para hoy. -¿Le importaría cambiarme de habitación? -Tenemos reservadas dos habitaciones con teléfono, por favor. -Por favor, reservé una habitación doble con teléfono para hoy. -¿Hay aire acondicionado en mi habitación, por favor? -He reservado una habitación. -¿Le importaría llevarnos las bolsas de viaje a la estación? -He reservado una habitación doble tranquila a nombre de Enrique Velasco. -¿Podría bajar mis bolsas? -Por favor, ¿le importaría darnos la llave de nuestra habitación? -Por favor, quisiéramos una habitación doble para una noche. -Por favor, quisiera cambiarme a otra habitación con televisión. -¿Tienen libre alguna habitación tranquila? -He reservado una habitación con teléfono. -Por favor, lléveme los bultos a mi habitación. -¿Está apuntado el recibo del teléfono? -Lleve mi equipaje a la estación. -Por favor, ¿querrían llamarnos a nuestro taxi? -¿Cuánto cuesta una habitación individual con media pensión? -La habitación es demasiado fría. -¿Me podrían llevar el equipaje al autobús? -Hicimos una reserva, por favor. -¿Podrían hacer nuestra cuenta?, por favor. -He reservado una habitación tranquila doble a nombre de Silvia Quereda. -Me llamo Jesús Rivera. -Tengo hecha una reserva a nombre de Sergio Moliner. -¿Tiene una habitación libre? -Por favor, pídame un taxi. -Prepárenos nuestra cuenta de la habitación seiscientos nueve. -¿Les importaría bajar nuestro equipaje a la habitación? -¿Me podría pedir un taxi? -Me gustaría reservar una habitación hasta el veinticinco. -Por favor, quiero reservar una habitación doble con teléfono. -¿Podrían despertarme a las dos? -Por favor, ¿pueden ustedes despertarnos a las once? -¿Tiene alguna habitación libre? -La habitación es demasiado fría. -¿Podría llamarme a un taxi?, por favor. -¿Cuánto vale una habitación individual para un día con pensión completa? -Por favor, ¿podría despertarme a la una en punto? -Tengo la reserva de una habitación tranquila doble con televisión y vistas. -¿Podrían llevarnos nuestras maletas a la número ocho diecinueve, por favor? -Perdone. -Por favor, tengo reservada una habitación doble tranquila con tele y teléfono. -¿Podrían ustedes despertarme a las once menos cuarto? -Me he de ir esta tarde. -Se ha producido una equivocación en la cuenta. -Nos gustaría cambiarnos de habitación, por favor. -Me parece que se produjo una equivocación en la cuenta de la habitación ocho siete cuatro. -¿Cuánto vale una habitación individual incluyendo desayuno, por favor? -Por favor, ¿me podría usted despertar a las tres y media? -La habitación es muy fría. -¿Le importaría darme la llave de la habitación? -He reservado una habitación. -Me gustaría cambiar mi habitación. -Mi nombre es Sergio Iborra. -Por favor, tengo hecha una reserva a nombre del señor Miralles. -Tengo una reserva a nombre del señor Ortiz. -Tenemos que marcharnos el sábado siete de marzo. -Por favor, ¿nos puede despertar a las diez y media? -Querría que nos bajasen el equipaje a nuestra habitación, por favor. -¿Cuánto vale una habitación individual para una noche? -Por favor, ¿cuánto vale por día una habitación doble? -Quisiera una habitación doble. -Quiero que me dé la llave de la número trescientos treinta y tres. -Por favor, reservé una habitación. -¿Está incluido el desayuno? -La habitación es muy cara. -Nos vamos a ir el día trece de septiembre a las ocho menos cuarto de la mañana. -Por favor, ¿me puede despertar a las once y media? -¿Le importaría llevarme mis bolsas al autobús? -Vamos a marcharnos mañana a las cuatro. -Pienso que se ha producido un error en la cuenta. -¿Le importaría que echara un vistazo a alguna habitación? -Por favor, quiero que nos bajen los bultos a la habitación. -¿Me podrían dar las llaves de la habitación seiscientos trece, por favor? -Por favor, ¿cuánto cuesta una habitación doble incluyendo servicio de habitaciones? -Por favor, ¿me pueden detallar la cuenta de la habitación cero catorce? -La habitación está bien. -Hice una reserva. -He hecho la reserva de una habitación doble tranquila con teléfono y ducha. -Hemos de marcharnos mañana por la noche. -¿Hay televisión y aire acondicionado en la habitación? -Despiérteme mañana a las diez y media, por favor. -Por favor, denme la llave de la habitación número nueve uno cero. -Hemos reservado una habitación doble con baño para hoy. -¿Tiene aire acondicionado, teléfono y televisión? -Se han equivocado en nuestra cuenta. -¿Podría ver alguna habitación individual con aire acondicionado? -¿Cuánto vale una habitación doble? -¿Podría ver alguna habitación doble, por favor? -¿Hay alguna habitación tranquila libre? -Hice la reserva de una habitación con tele y teléfono a nombre de Roberto Arnau. -He reservado una habitación con televisión y baño a nombre de Gerardo Ródenas. -Por favor, desearía una habitación. -Tenemos reservada una habitación. -Nos vamos a ir el jueves treinta de septiembre a las cuatro y media. -¿Podría ver una habitación doble con televisión? -¿Está todo? -¿Podrían ustedes darnos la llave de la habitación, por favor? -¿Pueden darme la llave de la habitación número setecientos setenta y tres? -¿Querrían hacernos la factura?, por favor. -Tenemos que irnos el jueves nueve de abril por la tarde. -Me voy a marchar hoy a las seis de la mañana. -Reservé una habitación doble y tranquila, por favor. -Por favor, reservé una habitación tranquila a nombre de Amelia Peinado. -Por favor, ¿me querría pedir un taxi para la habitación nueve ocho uno? -¿Cuánto cuesta una habitación doble, por favor? -Por favor, ¿querría pedirnos un taxi? -¿Podría detallar nuestra cuenta de la habitación nueve cuatro uno? -¿Está todo? -Por favor, ¿me pueden preparar mi cuenta? -He hecho la reserva de una habitación tranquila doble con vistas, ducha y teléfono a nombre de Carmelo Peris. -Por favor, ¿me podría dar usted la llave de la habitación? -Por favor, ¿podríamos ver otra habitación? -¿Nos pueden llevar las bolsas de viaje? -Por favor, desearía cambiarme a otra habitación con menos ruido. -¿Podrían despertarnos mañana a las nueve y cuarto? -Por favor, quisiera reservar una habitación. -¿Podríamos reservar dos habitaciones dobles para quince días? -¡No importa! -¿Podríamos reservar dos habitaciones? -¿Tienen alguna habitación doble libre? -Por favor, prepare la factura. -Por favor, desearía una habitación. -¿Está incluido el servicio de habitaciones? -Nos iremos esta noche. -Suban nuestro equipaje a nuestra habitación. -Me llamo Federico Viciano. -He reservado una habitación tranquila. -Por favor, tengo una reserva a nombre de la señora Guijarro. -¿Les importaría despertarnos a la una y media? -¿Tiene habitaciones individuales y tranquilas libres? -Nos tenemos que ir mañana. -Tengo reservada una habitación. -Por favor, hágame la cuenta. -¿Podría ver una habitación doble y tranquila, por favor? -Se ha equivocado en la factura. -Por favor, he hecho la reserva de una habitación con televisión, teléfono y vistas al mar a nombre de Asunción Espinosa. -¿Tiene aire acondicionado y agua caliente la habitación? -Quiero reservar una habitación hasta el próximo sábado. -¿Nos podrían llamar a un taxi para la habitación tres trece? -Quiero una habitación. -¿Pueden despertarme a la una menos cuarto? -Por favor, una habitación. -Me parece que se produjo un error en la cuenta. -Desearía una habitación para cinco noches. -Hemos reservado una habitación individual con teléfono a nombre del señor y la señora Santos, por favor. -¿Están incluidos todos los impuestos? -¿Está incluida la cuenta del teléfono? -Por favor, lleven nuestra bolsa de viaje a la habitación quinientos veintisiete. -He reservado una habitación tranquila. -Pida un taxi. -¿Nos pueden dar ustedes la llave de la habitación? -¿Nos podría llevar las bolsas al coche? -Tengo hecha una reserva a nombre de Susana Padilla. -Por favor, ¿pueden bajarnos los bultos a la habitación? -¿Le importaría bajar nuestro equipaje a la habitación, por favor? -Por favor, ¿cuánto cuesta una habitación doble por semana? -Despiértenme a la una y media, por favor. -Nos hemos de marchar mañana a las seis. -Por favor, tengo hecha una reserva. -La habitación es demasiado ruidosa. -Deseo que nos dé las llaves de la habitación siete dieciséis, por favor. -¿Está todo apuntado? -Tengo reservada una habitación doble, por favor. -Por favor, me gustaría reservar una habitación individual hasta el veintiuno. -Por favor, ¿me puede usted despertar mañana a las tres menos cuarto? -Despiértenos a la una, por favor. -Me tengo que ir el lunes dos de julio por la tarde. -Por favor, ¿tengo que rellenar alguna hoja de registro? -¿Podría ver alguna habitación? -Pienso que se produjo un error en la cuenta. -La habitación es muy cara. -Por favor, desearíamos que nos diera las llaves de la habitación cuatro tres. -¿Nos podría explicar la cuenta?, por favor. -Por favor, ¿nos podrían dar ustedes las llaves de la doscientos veintiséis? -¿Cuánto cuesta una habitación individual, por favor? -¿Podríamos reservar dos habitaciones para esta noche? -¿Nos podría despertar a la una, por favor? -Me parece que existe un problema en mi cuenta de la habitación nueve treinta y tres. -¡No! -Mi nombre es Marina Varela. -Por favor, deseo cambiarme de habitación. -Por favor, las llaves de la habitación número ocho veinte. -¿Están apuntados en el recibo los gastos? -Me voy hoy. -¿Les importaría darnos las llaves de la habitación? -Por favor, ¿tienen aire acondicionado las habitaciones? -¿Pueden subir el equipaje al coche? -¿Les importaría darnos la llave de la habitación? -¿Podría bajar mis maletas a mi habitación? -Deseo una habitación que tenga teléfono, vistas, televisión y baño a nombre de Marina Sáez. -Por favor, tengo la reserva de una habitación tranquila doble con vistas a la ciudad y baño a nombre de Ricardo Salsas. -Por favor, ¿tenemos que firmar la hoja de registro? -¿Nos podrían despertar mañana a las seis y media? -He de marcharme el viernes diecinueve de abril a las nueve de la noche. -Pienso que se ha equivocado en la cuenta. -Por favor, me gustaría cambiarme de habitación. -¿Está apuntado el teléfono? -Por favor, ¿me puede usted despertar a las dos menos cuarto? -Vamos a marcharnos el día siete a las once y media de la noche. -Desearía que nos despertasen a las tres y media, por favor. -Hemos de irnos mañana a las nueve. -¿Hay servicio de bar en la habitación? -Reservamos una habitación hasta el próximo jueves, por favor. -Por favor, ¿le importaría mostrarme una habitación doble con televisión? -Creo que hay un error en la factura de la habitación ocho tres ocho. -Por favor, háganos la cuenta de la habitación novecientos uno. -Quisiera reservar una habitación individual y tranquila hasta el uno de noviembre, por favor. -Por favor, he reservado una habitación tranquila individual con vistas a la montaña y televisión a nombre de la señora Gómez. -Por favor, he hecho una reserva. -¿Podría reservar una habitación? -Vamos a marcharnos el lunes diecinueve de abril por la mañana. -Tengo reservada una habitación tranquila. -Por favor, me hace mi cuenta. -Tengo que irme hoy a las once y media de la mañana. -¿Podrían preparar la cuenta?, por favor. -¿Tienen libre alguna habitación? -Creo que se ha equivocado en nuestra cuenta de la habitación quinientos quince. -Por favor, quisiera reservar una habitación. -¿Tienen una habitación libre? -¿Están anotados en el recibo los extras? -Tengo reservada una habitación. -Hemos hecho una reserva. -Por favor, ¿pueden pedirme mi taxi para la habitación setecientos dieciséis? -¿Puede despertarme a las cuatro y cuarto? -¿Están incluidos en la factura todos los impuestos? -¿Está incluido el desayuno? -Reservé una habitación a nombre del señor Gerardo Paches. -He reservado una habitación individual a nombre del señor Fernández. -Me iré el dieciocho a las siete menos cuarto de la mañana. -Por favor, ¿nos podría dar la llave de la seis cuatro siete? -¿Podríamos ver una habitación, por favor? -¿Pueden llevarnos nuestro equipaje al autobús? -Me marcharé el viernes uno a las tres. -¿Pueden ustedes bajarme mis bultos a la habitación? -¿Cuánto vale una habitación doble por día, por favor? -En la habitación hay mucho ruido. -¿Está incluida la cuenta del teléfono? -La habitación está bien. -Por favor, ¿hay agua caliente en las habitaciones? -¿Le importaría cambiarme de habitación? -¿Me sube mis bolsas de viaje al taxi? -Por favor, querríamos que nos despertaran a las ocho menos cuarto. -En la habitación hace mucho calor. -Queremos cambiar la habitación. -Súbame mis bolsas al autobús. -¿Tiene aire acondicionado mi habitación? -Despiérteme a las seis. -Voy a irme mañana a las tres menos cuarto de la tarde. -¿Hay habitaciones libres? -¿Podría cambiarnos a otra habitación con teléfono y televisión? -No importa. -¿Hay teléfono y televisión en la habitación? -He de marcharme hoy. -Vamos a marcharnos esta mañana. -Por favor, ¿le importaría mostrarnos una habitación con teléfono y televisión? -Lleve el equipaje. -¿Hay aire acondicionado en las habitaciones? -¿Podría subir mis bultos a mi habitación? -No estoy de acuerdo. -He reservado una habitación individual, por favor. -Quisiera reservar una habitación para un día, por favor. -¿Nos podría dar usted la llave de mi habitación? -¿Me puede dar la llave de la habitación, por favor? -Por favor, he hecho una reserva. -Me tengo que marchar mañana por la mañana. -¿Tengo que rellenar la hoja de registro, por favor? -¿Haría el favor de cambiarme de habitación? -Por favor, ¿puede darme las llaves de la cero tres cero? -¿Nos puede bajar nuestras bolsas de viaje a la habitación número cuatro veintitrés, por favor? -Deseo reservar una habitación individual y tranquila para una semana, por favor. -Por favor, explique la cuenta. -Por favor, ¿podrían pedirnos un taxi para la habitación siete veintinueve? -Por favor, tengo reservada una habitación para esta noche a nombre de Manuela Valls. -Por favor, querría que nos diera la llave de la habitación uno uno. -Por favor, ¿pueden bajar nuestros bultos a la habitación número cuatrocientos catorce? -Me gustaría cambiarme de habitación, por favor. -Por favor, hemos reservado una habitación individual y tranquila. -Por favor, suban mi bolsa de viaje a la habitación seiscientos treinta y uno. -¿Puede pedir un taxi? -Por favor, ¿puede pedir un taxi? -Por favor, ¿podrían hacer la cuenta de la habitación seiscientos veintiséis? -Dénos la llave de la número ocho catorce. -Quiero que me suba los bultos a la habitación, por favor. -Déme la llave de la habitación seis dieciocho, por favor. -¿Nos podrían dar la llave de la habitación número seis cuatro cero? -¿Podríamos ver una habitación tranquila, por favor? -¡Sí! -Llévenos nuestros bultos a la número ochocientos veintiséis, por favor. -¿Nos podría bajar el equipaje a la habitación número tres seis dos? -Por favor, he reservado una habitación tranquila hasta mañana a nombre de Gregorio Cabedo. -Nos marchamos hoy a las seis y media de la tarde. -Por favor, ¿le importaría llevar nuestras bolsas de viaje a la habitación número dos tres cinco? -Hicimos una reserva. -Por favor, ¿me podrían detallar la cuenta? -Me gustaría cambiarme a otra habitación con servicio de bar y aire acondicionado, por favor. -Quisiéramos que nos despertara a las once y cuarto. -Quisiéramos que nos despertase a las nueve en punto. -Reservé una habitación, por favor. -¿Podría cambiarnos a otra habitación con televisión? -Tengo que irme mañana a las diez en punto de la mañana. -¿Pueden explicar la cuenta? -Por favor, suba mi equipaje a mi habitación. -¿Nos pueden llevar nuestro equipaje al autobús? -¿Tenemos que firmar la hoja de registro? -Desearía que me despertara mañana a las diez, por favor. -Reservamos una habitación doble, por favor. -Vamos a marcharnos mañana a las cinco en punto de la tarde. -¿Hay aire acondicionado en la habitación? -Por favor, ¿le importaría darme la llave de la habitación trescientos treinta y tres? -Me voy a ir mañana a las cuatro y media de la tarde. -Nos tenemos que ir el lunes cuatro de abril a las doce y media de la mañana. -Por favor, ¿qué precio tiene una habitación doble por día con media pensión? -En la habitación hace demasiado calor. -¿Cuánto cuesta una habitación doble? -Por favor, ¿le importaría mostrarnos una habitación con teléfono? -¿Cuánto cuesta una habitación individual por día? -He de marcharme. -¿Me lleva mis bolsas? -Por favor, ¿cuánto cuesta una habitación doble para dieciocho días? -¿Nos puede despertar mañana a las cinco en punto? -¿Están incluidos en el recibo todos los gastos? -Tengo hecha la reserva de una habitación doble con baño y teléfono a nombre de Paloma Betoret. -Querría que me diera las llaves de la habitación dos treinta y nueve. -¿Podrían repasarme la cuenta? -Quiero que suba mi equipaje a mi habitación, por favor. -¿Nos pueden dar la llave de la habitación número uno cuatro cero, por favor? -¿Les importaría darme las llaves de la habitación, por favor? -¿Nos podría usted despertar a las seis menos cuarto? -Por favor, ¿cuánto vale una habitación individual? -¿Podríamos reservar dos habitaciones tranquilas para esta noche? -Por favor, tengo reservada una habitación individual y tranquila. -La habitación es muy cara. -Me parece que se produjo un problema en la cuenta. -Quiero una habitación doble para hoy. -Quiero una habitación individual. -Reservé una habitación a nombre de Alicia Morales, por favor. -Conforme. -Por favor, déme la llave de nuestra habitación. -Por favor, quisiéramos una habitación. -Me gustaría reservar dos habitaciones dobles, por favor. -Me voy a ir mañana a las diez menos cuarto de la noche. -Quisiera cambiarme de habitación. -La habitación es muy calurosa. -Hice la reserva de una habitación tranquila individual con buena vista del mar, teléfono y televisión. -¿Le importaría cambiarnos de habitación? -Tenemos reservada una habitación individual hasta el diecisiete. -Por favor, tengo una reserva. -¿Aceptan cheques de viaje? -Tenemos reservada una habitación, por favor. -¿Cuánto? -Por favor, ¿cuánto vale por día una habitación individual? -¿Tienen libre una habitación doble y tranquila? -Por favor, ¿nos quiere preparar la factura? -Por favor, ¿nos puede despertar a las cuatro? -Por favor, hemos reservado una habitación doble. -La habitación está bien. -Por favor, ¿podría repasar la cuenta? -Por favor, llame a mi taxi. -Quisiera que nos diesen la llave de mi habitación. -Quiero que nos despierte a las nueve y media. -Por favor, desearía una habitación tranquila doble que tenga teléfono, buena vista de la montaña, baño y televisión a nombre de la señora Berrueco. -Pídame mi taxi, por favor. -Por favor, pídame un taxi. -Me parece que se ha producido un error en la cuenta. -La habitación es demasiado fría. -Por favor, llámeme a mi taxi. -¿Cuánto cuesta por semana una habitación individual? -¿Le importaría que echara un vistazo a alguna habitación doble? -La habitación está bien. -Llévennos el equipaje a nuestra habitación. -¿Podría llevarme mis maletas? -Deseo una habitación para veintinueve días. -¿Tengo que rellenar la hoja de registro? -Deseo una habitación. -Quisiéramos una habitación individual hasta el próximo viernes. -Dennos la llave de la habitación número siete cero cinco. -¿Podría pedir un taxi? -Hemos de marcharnos el sábado catorce de mayo. -Prepare nuestra cuenta. -Me tengo que marchar mañana a las cinco menos cuarto de la tarde. -Quiero una habitación. -Existe una equivocación en la cuenta. -¿Tienen libre una habitación individual? -¿Hay una habitación libre? -Tenemos reservada una habitación individual a nombre de Luis Llopis. -Deseamos una habitación doble con ducha y aire acondicionado. -Por favor, tenenos hecha una reserva. -¿Está incluido el recibo del teléfono? -Por favor, déme la llave de la habitación. -Creo que se ha equivocado en nuestra cuenta. -Tengo la reserva de una habitación tranquila doble con televisión, cuarto de baño y una buena vista del mar. -¿Les importaría despertarme a las diez y media, por favor? -Por favor, haga la cuenta. -Tengo reservada una habitación a nombre de Jesús Barberá. -Quisiera una habitación para un día, por favor. -Por favor, ¿podría llamarme a un taxi? -Por favor, deseo que lleve mis bultos a mi habitación. -¿Hay televisión en mi habitación? -Por favor, reservé una habitación tranquila para veintiséis noches a nombre de Celestino Puig. -Me voy el dieciséis de septiembre a las doce menos cuarto de la mañana. -¿Cuánto cuesta por semana una habitación individual incluyendo desayuno? -Por favor, suban nuestra bolsa de viaje a la habitación número uno setenta y cuatro. -Por favor, prepare la cuenta. -Me he de ir el tres a las diez y cuarto de la mañana. -¿Tienen libre alguna habitación? -Por favor, tengo reservada una habitación tranquila con cuarto de baño, teléfono y televisión. -Despiértennos a las seis y cuarto. -¿Podríamos ver alguna habitación doble? -¿Podrían ustedes despertarnos a las seis y media? -Me parece que se han equivocado en la cuenta. -Quisiera que me diese la llave de la habitación, por favor. -Por favor, pida un taxi. -¿Tiene habitaciones libres? -Hemos reservado una habitación para diez semanas a nombre de Rosario Cornelles. -Por favor, reservamos una habitación tranquila. -¿Están incluidos en el recibo todos los extras? -Por favor, nos gustaría cambiarnos de habitación. -¿Podríamos ver una habitación? -¿Cuánto cuesta por semana una habitación doble incluyendo servicio de habitaciones, por favor? -Desearía una habitación con teléfono, por favor. -Por favor, reservamos dos habitaciones dobles con televisión. -¿Me pueden dar la llave de la habitación cero uno nueve? -Déme la llave de la habitación, por favor. -Existe un problema en la factura. -¿Está apuntada la cuenta del teléfono? -Voy a marcharme. -¿Me podría dar la llave de la quinientos uno? -Hemos de irnos hoy por la tarde. -¿Querrían pedirme un taxi?, por favor. -¿Podría bajar nuestro equipaje a recepción? -Por favor, ¿pueden ustedes subir mis bultos a la siete veintiocho? -¿Nos podría dar la llave de la habitación? -Hemos de irnos el día catorce por la noche. -Queremos una habitación para nueve noches, por favor. -Dénos las llaves de la habitación. -Deseo una habitación que tenga teléfono, televisión y ducha, por favor. -Reservé una habitación doble. -Por favor, me prepara la cuenta. -Lleven nuestras maletas a la habitación nueve veintiséis, por favor. -¿Cuánto vale una habitación individual por semana? -Por favor, quisiera una habitación para una semana. -Quisiera reservar una habitación, por favor. -Hice una reserva. -¿Podría hacer la factura? -Suba mi equipaje a la habitación número cero tres cinco. -La habitación es demasiado fría. -Nos prepara nuestra cuenta de la habitación tres cuatro uno, por favor. -Creo que hay un problema. -Por favor, deseo que nos dé las llaves de la habitación cero uno cero. -He hecho la reserva de una habitación individual a nombre de Dulce Gumbau. -Reservamos una habitación. -Por favor, ¿nos podrían dar la llave de la habitación número seis? -Por favor, ¿nos podrían hacer la cuenta? -¿Cuánto vale por día una habitación individual? -Por favor, hice una reserva a nombre de Roberto Vázquez. -¿Tienen libre alguna habitación con teléfono y televisión? -Súbanos nuestros bultos a la tres cuatro cero, por favor. -¿Me lleva el equipaje al coche? -Por favor, ¿qué precio tiene una habitación doble con pensión completa? -Quiero una habitación doble para hoy. -Tenemos que marcharnos hoy a las doce de la mañana. -Por favor, lleve mi bolsa de viaje a mi habitación. -Pida un taxi. -¿Cuánto vale una habitación individual? -¿Nos puede usted dar las llaves de mi habitación? -Por favor, deseo que me despierten mañana a las dos y media. -¿Les importaría subir mi equipaje a la habitación cuatro noventa, por favor? -¿Nos quieren hacer la factura? -¿Me querrían preparar la cuenta? -Por favor, he hecho una reserva. -Quiero una habitación individual y tranquila hasta mañana. -¿Podría usted llevarme mi maleta a mi habitación? -¿Les importaría despertarme a las tres y cuarto, por favor? -¿Puede darnos las llaves de la habitación número nueve dos siete? -He reservado una habitación individual con televisión, por favor. -Desearía que me llevasen mi bolsa a la habitación número ocho cero tres. -¿Cuánto cuesta una habitación individual? -¿Hay televisión en las habitaciones? -Por favor, quiero que lleve mi equipaje a la número siete tres tres. -Tengo una reserva. -¿Podría cambiarme de habitación? -¿Le importaría llevarnos nuestras maletas al taxi? -Por favor, ¿nos pueden despertar a las nueve? -Por favor, he reservado dos habitaciones. -¿Me pueden dar ustedes la llave de la habitación? -¡Adiós! -¿Puede darnos la llave de la habitación? -¿Pueden despertarme a la una? -¿Les importaría darnos la llave de la habitación, por favor? -Tenemos reservada una habitación individual y tranquila. -Deseo que nos dé la llave de la habitación novecientos once, por favor. -Por favor, ¿nos pueden ustedes dar la llave de mi habitación? -Voy a marcharme. -Tenemos que irnos esta tarde. -¿Le importaría subir mis bolsas de viaje al autobús? -¿Nos podrían dar la llave de la número cuatro siete uno, por favor? -Quiero una habitación individual y tranquila hasta mañana, por favor. -¿Podríamos reservar dos habitaciones? -Tengo hecha la reserva de una habitación con teléfono y ducha a nombre del señor Borillo. -Por favor, nos pide nuestro taxi. -Tenenos hecha una reserva. -Por favor, ¿me pueden preparar la cuenta de la habitación uno uno uno? -¿Le importaría llevar nuestro equipaje a nuestra habitación, por favor? -Por favor, quiero que lleve mi equipaje a la habitación seis catorce. -¿Le importaría mostrarnos alguna habitación tranquila, por favor? -Por favor, he hecho la reserva de una habitación con televisión, una buena vista y ducha a nombre de Virginia Pérez. -Pienso que hay una equivocación en la factura. -Me voy a ir el jueves por la mañana. -Reservamos una habitación individual y tranquila para treinta días a nombre de Lidia Llorens, por favor. -En la habitación hay mucho ruido. -Tenemos reservadas dos habitaciones tranquilas. -¿Le importaría mostrarnos una habitación? -Deseo una habitación con cuarto de baño, televisión y buena vista a nombre de Jorge Balaguer, por favor. -¿Tiene televisión? -¿Podríamos ver alguna habitación individual y tranquila? -Me parece que se produjo una equivocación en la cuenta. -En la habitación hace mucho calor. -Tengo que irme mañana a las dos y cuarto de la tarde. -¿Hay aire acondicionado en las habitaciones? -¿Nos pueden despertar a la una menos cuarto? -Reservamos una habitación doble para veintinueve días. -¿Podríamos reservar dos habitaciones individuales y tranquilas hasta el veintiocho? -¿Pueden llamarme a un taxi? -Deseo que me despierten a las siete y media. -Por favor, ¿me podría repasar la cuenta? -¿Tienen habitaciones tranquilas libres? -Me parece que se produjo un problema en la cuenta de la habitación nueve seis cuatro. -Denme las llaves de la habitación, por favor. -Llámenos a nuestro taxi para la habitación dos seis ocho. -¿Pueden darnos la llave de la habitación? -Déme la llave de la habitación cuatro uno tres. -¿Podrían llevarnos nuestro equipaje a recepción? -Por favor, desearíamos una habitación. -¿Cuánto vale por semana una habitación doble, por favor? -Por favor, ¿nos podrían despertar mañana a las nueve y media? -Reservamos una habitación individual para una noche a nombre de Micaela Carpio. -Por favor, pida nuestro taxi para la habitación ciento veintiocho. -¿Cuánto cuesta por día una habitación doble con pensión completa? -Despiértenme a la una y cuarto. -Por favor, tengo reservada una habitación con buena vista y teléfono. -¿Le importaría darnos las llaves de la habitación, por favor? -He de irme hoy a las ocho y cuarto de la mañana. -¿Nos puede pedir un taxi para la habitación cero diecinueve? -Creo que existe un error. -Queremos una habitación. -Tengo que irme el jueves veintiocho de octubre a las ocho en punto de la mañana. -¿Podría despertarme a las diez y media, por favor? -Quisiera que me despertara a la una, por favor. -Nos iremos el once. -¿Cuántas? -Tengo una reserva a nombre del señor Borrás. -¿Pueden llevarme las maletas al autobús? -Se produjo una equivocación en mi cuenta. -¿Podría ver una habitación, por favor? -Por favor, haga la factura de la habitación nueve catorce. -¿Podrían subir nuestros bultos a la cuatrocientos setenta y cuatro? -¿Tienen habitaciones tranquilas libres? -He reservado una habitación hasta el día uno de noviembre a nombre del señor Víctor Lobo, por favor. -¿Están apuntados los extras? -Tenemos reservada una habitación hasta el próximo domingo a nombre del señor Ángel Viciano. -¿Nos podría dar la llave de mi habitación, por favor? -Por favor, ¿pueden hacerme la factura de la habitación doscientos doce? -Por favor, ¿me podría bajar el equipaje a la habitación? -¿Podríamos reservar dos habitaciones individuales para hoy? -¿Tiene libre alguna habitación individual y tranquila? -Déme la llave de la habitación número tres diez. -Hemos reservado una habitación para hoy. -En la habitación hay demasiado ruido. -Nos hemos de ir el día uno por la mañana. -Desearía una habitación individual y tranquila, por favor. -Quisiéramos una habitación doble y tranquila. -Quiero que me despierten mañana a las doce menos cuarto. -Por favor, ¿podría usted despertarnos a la una y cuarto? -Vamos a irnos el día quince a las nueve menos cuarto de la mañana. -¿Están apuntados los impuestos? -Por favor, me llama a un taxi. -¿Me baja las bolsas de viaje a recepción? -Por favor, nos hace la cuenta de la habitación siete treinta y dos. -Tengo reservada una habitación para dieciocho días a nombre del señor y la señora Cabo. -¿Hay habitaciones libres con baño? -Por favor, ¿pueden hacernos la cuenta? -Reservé una habitación doble y tranquila con televisión para esta noche a nombre de la señorita Marina Miralles, por favor. -Creo que se produjo un problema en la cuenta de la habitación siete ocho seis. -¿Tienen habitaciones libres con teléfono? -Voy a marcharme hoy mismo por la tarde. -Hemos de marcharnos hoy mismo a las doce de la mañana. -Hemos de marcharnos el lunes por la noche. -Deseo una habitación con tele, ducha y teléfono. -Reservé una habitación para hoy a nombre del señor Rafael Soler. -¿Podría reservar una habitación doble? -Por favor, deseamos una habitación hasta el día veintiocho. -¿Tiene libre una habitación individual y tranquila? -Por favor, desearíamos cambiarnos a otra habitación con televisión y cuarto de baño. -¿Tiene habitaciones individuales y tranquilas libres? -¿Podría cambiarnos de habitación? -¿Están anotados todos los impuestos? -Pídame un taxi. -Tengo reservada una habitación a nombre del señor Ortiz. -Vamos a marcharnos el domingo a las nueve en punto de la mañana. -¿Nos querrían llamar a un taxi? -¡Muchas gracias! -¿Podría bajar nuestra maleta a nuestra habitación? -Tengo una reserva. -¿Pueden despertarme ustedes mañana a la una y cuarto? -Por favor, nos hace la cuenta de la habitación cuatrocientos diecinueve. -Existe una equivocación. -Por favor, hemos hecho una reserva. -Por favor, lleve mis bolsas de viaje a la habitación. -¿Tiene libre una habitación tranquila? -Tenemos reservadas dos habitaciones para una noche. -¿Me podrían repasar mi factura? -Por favor, tengo la reserva de una habitación tranquila. -¿Nos podría dar las llaves de la habitación número cinco quince, por favor? -Por favor, queremos reservar una habitación con aire acondicionado. -Tengo hecha la reserva de una habitación a nombre de Teresa Martínez. -Por favor, ¿me podría llamar a mi taxi? -He reservado una habitación con ducha, tele y teléfono a nombre de Emilio Paches. -Por favor, ¿nos podría dar la llave de la habitación cinco nueve? -¿Está todo incluido? -He hecho la reserva de una habitación a nombre del señor Ramos. -Me parece que existe una equivocación en mi cuenta de la habitación tres treinta y nueve. -¿Hay aire acondicionado en la habitación? -Nos hemos de marchar el catorce a la una y cuarto de la tarde. -Pienso que existe una equivocación en la cuenta de la habitación setecientos diecinueve. -Despiértenos a la una menos cuarto, por favor. -Pienso que existe una equivocación en la cuenta. -¿Podría subirnos el equipaje al taxi? -Deseamos reservar una habitación doble para esta noche. -¿Aceptan dinero en efectivo? -¿Me podrían subir el equipaje al coche? -Tengo reservada una habitación a nombre de Sergio Orenga. -Por favor, ¿podrían llamarme a un taxi para la habitación ciento noventa? -Por favor, querríamos que nos diera la llave de la habitación. -¿Hay teléfono en nuestra habitación? -¿Puede detallarnos la cuenta? -Por favor, ¿nos pueden ustedes dar la llave de mi habitación? -¿Hay una habitación individual libre? -Tenemos que irnos. -Por favor, ¿me pueden dar la llave de la número doscientos catorce? -¿Haría el favor de cambiarme de habitación? -¿Puede subirnos nuestro equipaje al coche? -Deseo una habitación tranquila. -Despiértenos mañana a las seis menos cuarto, por favor. -Quiero una habitación con televisión, teléfono y agua caliente hasta el día veintisiete. -Por favor, despiértenos a la una en punto. -Por favor, he reservado una habitación tranquila a nombre de Tomás Fernández. -¿Puede darnos la llave de la habitación número dos cinco nueve, por favor? -Por favor, despiértennos a la una y cuarto. -Por favor, he reservado una habitación. -Por favor, dennos las llaves de la habitación número seis cuatro uno. -Por favor, tengo hecha la reserva de una habitación con televisión y teléfono a nombre de la señorita Calleja. -Por favor, reservé una habitación con tele, una buena vista y teléfono. -Querría que nos despertaran mañana a las seis menos cuarto, por favor. -Quisiera cambiarme de habitación. -Por favor, queremos cambiar la habitación. -¿Tienen habitaciones dobles libres? -Desearía una habitación. -Quisiéramos una habitación. -Por favor, desearía que me dieran la llave de nuestra habitación. -Tengo que marcharme el día treinta de septiembre. -¿Hay una habitación libre? -Por favor, ¿me querrían preparar la factura? -Querría una habitación con televisión y teléfono, por favor. -Por favor, he reservado una habitación con teléfono. -Despiértenos mañana a las seis en punto, por favor. -Hemos reservado una habitación tranquila, por favor. -Tengo una reserva, por favor. -¿Podría ver alguna habitación individual y tranquila, por favor? -Por favor, ¿cuánto vale una habitación individual? -Lléveme las bolsas al taxi. -Deseamos cambiar nuestra habitación. -¿Nos puede llevar las bolsas a la estación? -¿Le importaría mostrarme otra habitación? -Por favor, deseo reservar una habitación. -He hecho una reserva. -¿Qué precio tiene una habitación individual para nueve días, por favor? -Tenemos que irnos mañana a la una. -Por favor, lleven mis bolsas a la habitación cero veintinueve. -¿Me podrían llevar mis bolsas al autobús? -Tenemos reservada una habitación para una noche a nombre del señor Federico Cornelles. -Hice una reserva. -Me parece que se ha producido un problema. -¿Puede preparar la factura de la habitación cinco uno uno?, por favor. -La habitación es muy calurosa. -¿Podría ver alguna habitación individual y tranquila? -Me llama a un taxi. -Despiértenme mañana a las cinco. -¿Está incluido el servicio de habitaciones? -Por favor, ¿nos podría dar las llaves de la habitación? -Me voy a marchar el viernes veintinueve de mayo a las nueve en punto de la noche. -Quiero reservar una habitación individual y tranquila para esta semana, por favor. -¿Cuánto cuesta una habitación doble con pensión completa? -¿Nos podría despertar a las siete y media? -Reservamos una habitación individual. -Baje nuestras bolsas de viaje a recepción. -Hemos reservado una habitación hasta el dos de febrero. -Me gustaría reservar una habitación para esta noche, por favor. -¿Les importaría darme la llave de mi habitación? -Por favor, ¿le importaría despertarme a las ocho y cuarto? -¿Nos podrían despertar a las seis y media? -Queremos que nos den la llave de la habitación. -Por favor, tengo la reserva de una habitación tranquila individual a nombre de Enrique Díaz. -¿Tienen una habitación libre con agua caliente y aire acondicionado? -¿Tiene teléfono y aire acondicionado la habitación? -Nos tenemos que ir el día veintisiete a las nueve en punto. -Por favor, ¿me querrían llamar a un taxi? -¿Les importaría despertarnos mañana a las nueve menos cuarto? -Reservamos una habitación con aire acondicionado hasta pasado mañana, por favor. -¿Nos puede pedir un taxi?, por favor. -Tenemos reservada una habitación doble. -Desearía que llevase mis bolsas a mi habitación, por favor. -Deseo una habitación doble. -Me marcho mañana a las doce en punto de la mañana. -¿Hay aire acondicionado y servicio de bar en las habitaciones? -Por favor, reservamos dos habitaciones. -¿Hay caja fuerte en las habitaciones? -¿Tienen habitaciones tranquilas libres? -Por favor, ¿me puede dar usted las llaves de la habitación uno uno tres? -Por favor, suba mi maleta a mi habitación. -Por favor, tengo reservada una habitación doble con aire acondicionado, teléfono y televisión. -Quisiera reservar una habitación. -¿Podría ver otra habitación? -¿Tiene habitaciones dobles libres? -¿Están incluidos todos los impuestos? -Por favor, despiértenos mañana a las dos. -¿Nos podría dar la llave de la habitación, por favor? -Súbanos las maletas al coche. -¿Cuánto cuesta una habitación individual? -Me voy a ir el viernes once de abril a las ocho y cuarto de la noche. -Por favor, quiero que nos dé la llave de mi habitación. -¿Cuánto cuesta una habitación doble por semana? -Por favor, hice la reserva de una habitación doble tranquila con cuarto de baño y vistas a la ciudad a nombre de Arturo Mira. -Por favor, he reservado una habitación. -¿Hay televisión en la habitación, por favor? -Pienso que se ha producido un problema en mi cuenta de la habitación seis cinco. -¿Cuánto cuesta una habitación individual, por favor? -Me hace la cuenta de la habitación uno cuatro siete. -Por favor, ¿hay agua caliente en la habitación? -¿Tienen alguna habitación individual libre? -¿Cuánto cuesta una habitación individual para un día? -¿Nos puede llamar a un taxi para la habitación tres seis? -¿Cuánto vale por día una habitación doble? -Por favor, tenenos hecha una reserva. -He reservado una habitación tranquila doble a nombre de Paloma Montero. -Deseamos cambiarnos a otra habitación con cuarto de baño, por favor. -¿Me podría despertar a las once en punto, por favor? -¿Podrían llamarnos a un taxi para la habitación nueve tres seis? -Vamos a marcharnos el día veintiséis de septiembre a las siete y cuarto. -Por favor, deseo que me den la llave de la habitación. -Por favor, ¿me podría pedir un taxi? -¿Aceptaría dinero en efectivo? -Tengo la reserva de una habitación a nombre de Francisco Botella. -Prepáreme la factura. -Me gustaría reservar una habitación individual y tranquila con teléfono hasta pasado mañana, por favor. -¿Podría pedirnos un taxi? -¿Hay televisión en las habitaciones? -¿Cuánta? -¡Conforme! -Por favor, ¿podría prepararnos la cuenta de la habitación dos seis tres? -¿Hay teléfono en las habitaciones? -¿Qué hora es? -Mi nombre es Jesús González. -Por favor, tengo hecha una reserva a nombre de Teresa Ibáñez. -He de marcharme hoy mismo a las once en punto de la mañana. -Desearía que me bajase los bultos a la habitación doce. -¿Dónde? -Por favor, ¿nos podría pedir un taxi para la habitación seis catorce? -Por favor, ¿les importaría despertarme mañana a las cuatro y media? -Por favor, nos gustaría cambiarnos a otra habitación menos ruidosa. -Creo que se ha equivocado en la cuenta. -¿Está incluido el teléfono? -Una habitación a nombre de Virginia Álvarez, por favor. -Tenemos que marcharnos hoy mismo por la noche. -En la habitación hace mucho calor. -Por favor, ¿me podrían preparar mi factura? -¿Podría ver alguna habitación, por favor? -Hemos reservado una habitación doble. -¿Aceptarían tarjetas de crédito? -Me he de marchar el jueves dieciséis a las nueve y media de la noche. -Por favor, suba mi maleta a mi habitación. -Deseo una habitación individual. -¿Hay habitaciones dobles y tranquilas libres? -Deseamos que baje nuestras maletas a la habitación. -¿Tienen libre una habitación? -Nos vamos a marchar el sábado. -Por favor, tengo hecha la reserva de una habitación con televisión y teléfono a nombre de Inmaculada Colomer. -Súbanme mis maletas a la habitación número dos ocho, por favor. -¿Les importaría despertarnos a las cuatro y media, por favor? -Hay un error en la cuenta de la habitación setecientos diecinueve. -¿Querría pedirnos nuestro taxi para la habitación dos tres uno? -Por favor, ¿podrían subirnos nuestra bolsa de viaje a la habitación? -¿Les importaría llevar nuestros bultos a nuestra habitación? -¿Podría cambiarme a otra habitación menos fría? -Hemos reservado una habitación. -¿Le importaría que echáramos un vistazo a alguna habitación? -¿Cuántas? -Déme las llaves de nuestra habitación. -¿Podría llamarme a un taxi? -Desearía que me llevara mi maleta a la habitación número setecientos tres, por favor. -Por favor, tenenos hecha una reserva. -¿Podría cambiarme a otra habitación más cálida? -Adiós. -Por favor, lléveme mi maleta a la habitación. -Por favor, ¿podrían llevarme ustedes mis bultos a la habitación setecientos veintitrés? -¿Hay alguna habitación doble libre? -Por favor, dénos la llave de la habitación. -Pienso que se ha producido una equivocación en la cuenta. -Despiértenos mañana a la una en punto, por favor. -¿Cuándo? -Me llamo Emilio Navarro. -¿Le importaría subir nuestras bolsas a nuestra habitación, por favor? -Existe un error en nuestra cuenta. -¿Hay teléfono en la habitación? -Por favor, tengo la reserva de una habitación con teléfono, televisión y baño a nombre de Isabel Cabedo. -¿Me podría despertar mañana a las doce, por favor? -Por favor, ¿podríamos ver otra habitación? -¿Me podrían hacer la cuenta? -¿Podría reservar una habitación? -He de marcharme mañana a las seis. -Queremos que nos dé la llave de la habitación número novecientos veintiuno, por favor. -Lleve nuestras bolsas a nuestra habitación. -¿Podría cambiarnos a otra habitación con menos ruido? -Pienso que existe una equivocación en mi cuenta. -He de marcharme el día ocho de septiembre. -He reservado una habitación doble a nombre de la señora Inmaculada Colomer. -¿Podríamos reservar dos habitaciones? -Bajen nuestros bultos a la habitación número cuatrocientos dieciocho. -Me tengo que marchar el trece a las cuatro y cuarto. -¿Tiene libre una habitación individual? -Reservamos dos habitaciones, por favor. -Pienso que existe un error. -¿Haría el favor de cambiarnos a otra habitación con menos ruido? -Dennos las llaves de la habitación nueve nueve dos, por favor. -Por favor, deseo una habitación. -He hecho la reserva de una habitación con teléfono, vistas a la montaña, baño y tele a nombre de Ricardo Vilanova. -Deseo que me suban los bultos a mi habitación. -Tengo hecha una reserva, por favor. -¿Nos podría despertar mañana a las cuatro? -Me parece que se produjo un error en la cuenta. -He de irme mañana. -¿Le importaría darnos la llave de la habitación? -Desearía una habitación hasta el próximo martes. -¿Dónde? -Por favor, ¿les importaría darme la llave de la habitación quinientos veintidós? -Quiero reservar una habitación. -Lléveme mi equipaje a la estación. -¿Me lleva las bolsas? -¿Podrían darnos la llave de mi habitación? -Desearíamos una habitación doble con ducha para un día. -¿Tiene una habitación doble libre? -Se ha producido una equivocación en la cuenta. -Desearía reservar una habitación para siete días. -Tenemos reservada una habitación individual con televisión, teléfono y aseo hasta mañana a nombre de José Herrero, por favor. -¿Están anotados en la cuenta los extras? -Hemos reservado una habitación doble. -La habitación es muy fría. -Tengo reservada una habitación doble para hoy. -¡Buenos días! -¿Pueden llevarme las bolsas de viaje al coche? -¿Podría ver alguna habitación doble y tranquila con televisión, por favor? -¿Están incluidos en la cuenta todos los impuestos? -Hay una equivocación en la cuenta. -¿Le importaría subirme mi equipaje al autobús? -Me marcharé el veintisiete de marzo a las dos y media de la tarde. -Por favor, reservé una habitación tranquila doble con baño, una buena vista y televisión a nombre de Andrés Sanz. -Por favor, llámenos a un taxi. -¿Tenemos que firmar alguna hoja de registro? -He hecho la reserva de una habitación a nombre de Julia Vilanova. -¿Hay alguna habitación tranquila libre? -Me llamo Gregorio Pitarch. -¿Cómo? -Llámeme a mi taxi. -Creo que hay un error en la cuenta. -¿Tiene libre una habitación con aseo, aire acondicionado y televisión? -¿Querría llamarnos a un taxi?, por favor. -De nada. -Reservamos una habitación a nombre de Juan Rubio. -¿Podría reservar una habitación con aire acondicionado para esta semana? -Nos marchamos hoy mismo a las diez y cuarto de la noche. -En la habitación hace demasiado frío. -Por favor, tengo una reserva. -Nos iremos mañana a las tres menos cuarto de la tarde. -Deseamos que nos despierten a las once, por favor. -Quisiera que me diesen las llaves de la habitación número seis tres ocho. -¿Nos pueden preparar la factura?, por favor. -Reservamos una habitación individual, por favor. -Queremos reservar una habitación, por favor. -¿Le importaría llevar mi equipaje a recepción? -¿Me podría llamar a un taxi? -¿Tiene aire acondicionado la habitación? -Me he de ir mañana a las diez y cuarto de la noche. -He reservado una habitación individual. -¿Tiene alguna habitación individual libre? -Nos tenemos que ir el día cuatro. -¿Puedo pagar la cuenta en efectivo? -¿Nos podría usted dar las llaves de nuestra habitación? -Quiero una habitación individual y tranquila para hoy, por favor. -¿Tenemos que firmar la hoja de registro? -Por favor, tenenos hecha una reserva. -¿Le importaría mostrarme alguna habitación con baño, por favor? -Despiérteme mañana a las tres menos cuarto, por favor. -Querría una habitación que tenga buena vista de la montaña y ducha, por favor. -He reservado una habitación. -¿Tenemos que rellenar la hoja de registro? -¿Me pueden despertar a las dos en punto, por favor? -¿Nos puede despertar a las diez, por favor? diff --git a/samples/machine_translation.ipynb b/samples/machine_translation.ipynb deleted file mode 100644 index 9698151cb..000000000 --- a/samples/machine_translation.ipynb +++ /dev/null @@ -1,467 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Machine Translation Tutorial\n", - "\n", - "Machine provides a general framework for machine translation engines. It currently provides implementations for rule-based MT, statistical MT (SMT), and neural MT (NMT). All MT engines implement the same interfaces, which provides a high level of extensibility for calling applications.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
Installed Packages
  • Nito.AsyncEx, 5.1.2
  • SIL.Scripture, 12.0.1
  • Thot, 3.4.1
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "#r \"nuget:SIL.Scripture,12.0.1\"\n", - "#r \"nuget:Thot\"\n", - "#r \"nuget:Nito.AsyncEx\"\n", - "#r \"../src/SIL.Machine/bin/Debug/netstandard2.0/SIL.Machine.dll\"\n", - "#r \"../src/SIL.Machine.Morphology.HermitCrab/bin/Debug/netstandard2.0/SIL.Machine.Morphology.HermitCrab.dll\"\n", - "#r \"../src/SIL.Machine.Translation.Thot/bin/Debug/netstandard2.0/SIL.Machine.Translation.Thot.dll\"\n", - "\n", - "void Write(string text)\n", - "{\n", - " Console.Write(text);\n", - "}\n", - "\n", - "void WriteLine(string text = \"\")\n", - "{\n", - " Console.Write(text + \"\\n\");\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Statistical Machine Translation\n", - "\n", - "Machine provides a phrase-based statistical machine translation engine that is based on the [Thot](https://github.com/sillsdev/thot) library. The SMT engine implemented in Thot is unique, because it supports incremental training and interactive machine translation (IMT). Let's start by training an SMT model. MT models implement the `ITranslationModel` interface. SMT models are trained using a parallel text corpus, so the first step is to create a `ParallelTextCorpus`.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - "using SIL.Machine.Corpora;\n", - "using SIL.Machine.Tokenization;\n", - "\n", - "var sourceCorpus = new TextFileTextCorpus(\"data/sp.txt\");\n", - "var targetCorpus = new TextFileTextCorpus(\"data/en.txt\");\n", - "var parallelCorpus = sourceCorpus.AlignRows(targetCorpus);" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Trainers are responsible for training MT models. A trainer can be created either using the constructor or using the `CreateTrainer` method on the `ITranslationModel` interface. Creating a trainer by constructor is useful if you are training a new model. The `CreateTrainer` method is useful when you are retraining an existing model. In this example, we are going to construct the trainer directly. Word alignment is at the core of SMT. In this example, we are going to use HMM for word alignment.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Training model... done.\n", - "Saving model... done.\n" - ] - } - ], - "source": [ - "using System.IO;\n", - "using SIL.Machine.Translation.Thot;\n", - "using SIL.Machine.Utils;\n", - "\n", - "var tokenizer = new LatinWordTokenizer();\n", - "Directory.CreateDirectory(\"out/sp-en\");\n", - "File.Copy(\"data/smt.cfg\", \"out/sp-en/smt.cfg\", overwrite: true);\n", - "{\n", - " using var trainer = new ThotSmtModelTrainer(ThotWordAlignmentModelType.Hmm, parallelCorpus, \"out/sp-en/smt.cfg\")\n", - " {\n", - " SourceTokenizer = tokenizer,\n", - " TargetTokenizer = tokenizer,\n", - " LowercaseSource = true,\n", - " LowercaseTarget = true\n", - " };\n", - "\n", - " Write(\"Training model...\");\n", - " await trainer.TrainAsync();\n", - " WriteLine($\" done.\");\n", - " Write(\"Saving model...\");\n", - " await trainer.SaveAsync();\n", - " WriteLine($\" done.\");\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In order to fully translate a sentence, we need to perform pre-processing steps on the source sentence and post-processing steps on the target translation. Here are the steps to fully translate a sentence:\n", - "\n", - "1. Tokenize the source sentence.\n", - "2. Lowercase the source tokens.\n", - "3. Translate the sentence.\n", - "4. Truecase the target tokens.\n", - "5. Detokenize the target tokens into a sentence.\n", - "\n", - "Truecasing is the process of properly capitalizing a lowercased sentence. Luckily, Machine provides a statistical truecaser that can learn the capitalization rules for a language. The next step is train the truecaser model.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - "using SIL.Machine.Translation;\n", - "\n", - "{\n", - " var truecaser = new UnigramTruecaser(\"out/sp-en/en.truecase.txt\");\n", - " using var trainer = truecaser.CreateTrainer(targetCorpus);\n", - " await trainer.TrainAsync();\n", - " await trainer.SaveAsync();\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now that we have a trained SMT model and a trained truecasing model, we are ready to translate sentences. First, We need to load the SMT model. The model can be used to translate sentences using the `TranslateAsync` method.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "I would like to book a room until tomorrow.\n" - ] - } - ], - "source": [ - "var truecaser = new UnigramTruecaser(\"out/sp-en/en.truecase.txt\");\n", - "var detokenizer = new LatinWordDetokenizer();\n", - "\n", - "{ \n", - " using var model = new ThotSmtModel(ThotWordAlignmentModelType.Hmm, \"out/sp-en/smt.cfg\")\n", - " {\n", - " SourceTokenizer = tokenizer,\n", - " TargetTokenizer = tokenizer,\n", - " TargetDetokenizer = detokenizer,\n", - " Truecaser = truecaser,\n", - " LowercaseSource = true,\n", - " LowercaseTarget = true\n", - " };\n", - "\n", - " var result = await model.TranslateAsync(\"Desearía reservar una habitación hasta mañana.\");\n", - " WriteLine(result.Translation);\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "`ThotSmtModel` also supports interactive machine translation. Under this paradigm, the engine assists a human translator by providing translations suggestions based on what the user has translated so far. This paradigm can be coupled with incremental training to provide a model that is constantly learning from translator input. Models and engines must implement the `IInteractiveTranslationModel` and `IInteractiveTranslationEngine` interfaces to support IMT. The IMT paradigm is implemented in the `InteractiveTranslator` class. The `ApproveAsync` method on `InteractiveTranslator` performs incremental training using the current prefix. Suggestions are generated from translations using a class that implements the `ITranslationSuggester` interface.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Source: Hablé con recepción.\n", - "Suggestion: [With reception]\n", - "Suggestion: I spoke [with reception]\n", - "Suggestion: I spoke with reception. []\n", - "\n", - "Source: Hablé hasta cinco en punto.\n", - "Suggestion: [I spoke until five o'clock]\n", - "Suggestion: I spoke until five o'clock. []\n" - ] - } - ], - "source": [ - "var suggester = new PhraseTranslationSuggester();\n", - "string GetCurrentSuggestion(InteractiveTranslator translator)\n", - "{\n", - " var suggestion = suggester.GetSuggestions(1, translator).FirstOrDefault();\n", - " var suggestionText = suggestion is null ? \"\" : detokenizer.Detokenize(suggestion.TargetWords);\n", - " if (translator.Prefix.Length == 0)\n", - " suggestionText = suggestionText.Capitalize();\n", - " var prefixText = translator.Prefix.Trim();\n", - " if (prefixText.Length > 0)\n", - " prefixText = prefixText + \" \";\n", - " return $\"{prefixText}[{suggestionText}]\";\n", - "}\n", - "\n", - "{\n", - " using var model = new ThotSmtModel(ThotWordAlignmentModelType.Hmm, \"out/sp-en/smt.cfg\")\n", - " {\n", - " SourceTokenizer = tokenizer,\n", - " TargetTokenizer = tokenizer,\n", - " TargetDetokenizer = detokenizer,\n", - " Truecaser = truecaser,\n", - " LowercaseSource = true,\n", - " LowercaseTarget = true\n", - " };\n", - " var factory = new InteractiveTranslatorFactory(model)\n", - " {\n", - " TargetTokenizer = tokenizer,\n", - " TargetDetokenizer = detokenizer\n", - " };\n", - "\n", - " var sourceSentence = \"Hablé con recepción.\";\n", - " WriteLine($\"Source: {sourceSentence}\");\n", - " var translator = await factory.CreateAsync(sourceSentence);\n", - "\n", - " var suggestion = GetCurrentSuggestion(translator);\n", - " WriteLine($\"Suggestion: {suggestion}\");\n", - "\n", - " translator.AppendToPrefix(\"I spoke \");\n", - " suggestion = GetCurrentSuggestion(translator);\n", - " WriteLine($\"Suggestion: {suggestion}\");\n", - "\n", - " translator.AppendToPrefix(\"with reception.\");\n", - " suggestion = GetCurrentSuggestion(translator);\n", - " WriteLine($\"Suggestion: {suggestion}\");\n", - " await translator.ApproveAsync(alignedOnly: false);\n", - " WriteLine();\n", - "\n", - " sourceSentence = \"Hablé hasta cinco en punto.\";\n", - " WriteLine($\"Source: {sourceSentence}\");\n", - " translator = await factory.CreateAsync(sourceSentence);\n", - "\n", - " suggestion = GetCurrentSuggestion(translator);\n", - " WriteLine($\"Suggestion: {suggestion}\");\n", - "\n", - " translator.AppendToPrefix(\"I spoke until five o'clock.\");\n", - " suggestion = GetCurrentSuggestion(translator);\n", - " WriteLine($\"Suggestion: {suggestion}\");\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Rule-based Machine Translation\n", - "\n", - "Machine provides an implementation of a simple, transfer-based MT engine. Transfer-based MT consists of three steps:\n", - "\n", - "1. Analysis: source words are segmented into morphemes.\n", - "2. Transfer: source morphemes are converted to the equivalent target morphemes.\n", - "3. Synthesis: the target morphemes are combined into target words.\n", - "\n", - "The `TransferEngine` class implements this process. HermitCrab, a rule-based morphological parser, can be used to perform the analysis and synthesis steps. HermitCrab parser implementation is provided in the [SIL.Machine.Morphology.HermitCrab](https://www.nuget.org/packages/SIL.Machine.Morphology.HermitCrab/) package. In this example, the transfer is performed using simple gloss matching.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "God created the world.\n" - ] - } - ], - "source": [ - "using SIL.Machine.Morphology.HermitCrab;\n", - "\n", - "var hcTraceManager = new TraceManager();\n", - "\n", - "Language srcLang = XmlLanguageLoader.Load(\"data/sp-hc.xml\");\n", - "var srcMorpher = new Morpher(hcTraceManager, srcLang);\n", - "\n", - "Language trgLang = XmlLanguageLoader.Load(\"data/en-hc.xml\");\n", - "var trgMorpher = new Morpher(hcTraceManager, trgLang);\n", - "\n", - "var transferer = new SimpleTransferer(new GlossMorphemeMapper(trgMorpher));\n", - "\n", - "{\n", - " using var transferEngine = new TransferEngine(srcMorpher, transferer, trgMorpher)\n", - " {\n", - " SourceTokenizer = tokenizer,\n", - " TargetDetokenizer = detokenizer,\n", - " Truecaser = truecaser,\n", - " LowercaseSource = true\n", - " };\n", - "\n", - " var result = await transferEngine.TranslateAsync(\"Dios creó el mundo.\");\n", - " WriteLine(result.Translation.Capitalize());\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Hybrid Machine Translation\n", - "\n", - "Machine includes a hybrid machine translation approach that allows you to merge the translation results from a rule-based engine and data-driven engine. The translation from the data-drive engine is the base translation. If there are any words/phrases in the base translation that have a low score, then they will be replaced by the translations from the rule-based engine. This hybrid approach is implemented in the `HybridTranslationEngine` class.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "SMT: Please make out two cuentas.\n", - "Transfer: Por favor, haga dos bills.\n", - "Hybrid: Please make out two bills.\n" - ] - } - ], - "source": [ - "{\n", - " using var smtModel = new ThotSmtModel(ThotWordAlignmentModelType.Hmm, \"out/sp-en/smt.cfg\")\n", - " {\n", - " SourceTokenizer = tokenizer,\n", - " TargetTokenizer = tokenizer,\n", - " TargetDetokenizer = detokenizer,\n", - " Truecaser = truecaser,\n", - " LowercaseSource = true,\n", - " LowercaseTarget = true\n", - " };\n", - "\n", - " using var transferEngine = new TransferEngine(srcMorpher, transferer, trgMorpher)\n", - " {\n", - " SourceTokenizer = tokenizer,\n", - " TargetDetokenizer = detokenizer,\n", - " Truecaser = truecaser,\n", - " LowercaseSource = true\n", - " };\n", - "\n", - " using var hybridEngine = new HybridTranslationEngine(smtModel, transferEngine)\n", - " {\n", - " TargetDetokenizer = detokenizer\n", - " };\n", - "\n", - " var sourceSentence = \"Por favor, haga dos cuentas.\";\n", - " var result = await smtModel.TranslateAsync(sourceSentence);\n", - " WriteLine($\"SMT: {result.Translation.Capitalize()}\");\n", - "\n", - " result = await transferEngine.TranslateAsync(sourceSentence);\n", - " WriteLine($\"Transfer: {result.Translation.Capitalize()}\");\n", - "\n", - " result = await hybridEngine.TranslateAsync(sourceSentence);\n", - " WriteLine($\"Hybrid: {result.Translation.Capitalize()}\");\n", - "}" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".NET (C#)", - "language": "C#", - "name": ".net-csharp" - }, - "language_info": { - "name": "C#" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/samples/tokenization.ipynb b/samples/tokenization.ipynb deleted file mode 100644 index c84847d08..000000000 --- a/samples/tokenization.ipynb +++ /dev/null @@ -1,419 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Tokenization Tutorial\n", - "\n", - "There are many NLP methods that require tokenized data as input, such as machine translation and word alignment. In this notebook, we will show how to use the different tokenizers and detokenizers that are available in Machine. Tokenizers implement either the `ITokenizer` interface or the `IRangeTokenizer` interface. `ITokenizer` classes are used to segment a sequence into tokens. `IRangeTokenizer` classes return ranges that mark where each each token occurs in the sequence. Detokenizers implement the `IDetokenizer` interface.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
Installed Packages
  • SIL.Scripture, 12.0.1
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "#r \"nuget:SIL.Scripture,12.0.1\"\n", - "#r \"../src/SIL.Machine/bin/Debug/netstandard2.0/SIL.Machine.dll\"\n", - "#r \"../src/SIL.Machine.Tokenization.SentencePiece/bin/Debug/netstandard2.0/SIL.Machine.Tokenization.SentencePiece.dll\"\n", - "\n", - "void WriteLine(string text = \"\")\n", - "{\n", - " Console.Write(text + \"\\n\");\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tokenizing text\n", - "\n", - "Let's start with a simple, whitespace tokenizer. This tokenizer is used to split a string at whitespace. This tokenizer is useful for text that has already been tokenized.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This | is | a | test | .\n" - ] - } - ], - "source": [ - "using SIL.Machine.Tokenization;\n", - "\n", - "var tokenizer = new WhitespaceTokenizer();\n", - "var tokens = tokenizer.Tokenize(\"This is a test .\");\n", - "WriteLine(string.Join(\" | \", tokens));" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Machine contains general tokenizers that can be used to tokenize text from languages with a Latin-based script. A word tokenizer and a sentence tokenizer are available.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Integer | scelerisque | efficitur | dui | , | eu | tincidunt | erat | posuere | in | .\n", - "Curabitur | vel | finibus | mi | .\n" - ] - } - ], - "source": [ - "var sentenceTokenizer = new LatinSentenceTokenizer();\n", - "var sentences = sentenceTokenizer.Tokenize(\n", - " \"Integer scelerisque efficitur dui, eu tincidunt erat posuere in. Curabitur vel finibus mi.\");\n", - "var wordTokenizer = new LatinWordTokenizer();\n", - "WriteLine(string.Join(\"\\n\", sentences.Select(s => string.Join(\" | \", wordTokenizer.Tokenize(s)))));" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Most tokenizers implement the `IRangeTokenizer` interface. These tokenizers have an additional method, `TokenizeAsRanges`, that returns ranges that mark the position of all tokens in the original string.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[\"][This] [is] [a] [test][,] [also][.][\"]\n" - ] - } - ], - "source": [ - "var wordTokenizer = new LatinWordTokenizer();\n", - "var sentence = \"\\\"This is a test, also.\\\"\";\n", - "var ranges = wordTokenizer.TokenizeAsRanges(sentence);\n", - "var output = \"\";\n", - "var prev_end = 0;\n", - "foreach (var range in ranges)\n", - "{\n", - " output += sentence.Substring(prev_end, range.Start - prev_end);\n", - " output += $\"[{sentence.Substring(range.Start, range.Length)}]\";\n", - " prev_end = range.End;\n", - "}\n", - "WriteLine(output + sentence.Substring(prev_end));" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "There are some languages that do not delimit words with spaces, but instead delimit sentences with spaces. In these cases, it is common practice to use zero-width spaces to explicitly mark word boundaries. This is often done for Bible translations. Machine contains a word tokenizer that is designed to properly deal with text use zero-width space to delimit words and spaces to delimit sentences. Notice that the space is preserved, since it is being used as punctuation to delimit sentences.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Lorem | Ipsum | Dolor | Sit | Amet | Consectetur | | Adipiscing | Elit | Sed\n" - ] - } - ], - "source": [ - "var wordTokenizer = new ZwspWordTokenizer();\n", - "var tokens = wordTokenizer.Tokenize(\"Lorem​Ipsum​Dolor​Sit​Amet​Consectetur Adipiscing​Elit​Sed\");\n", - "WriteLine(string.Join(\" | \", tokens));" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Subword tokenization has become popular for use with deep learning models. Machine provides a [SentencePiece](https://github.com/google/sentencepiece) tokenizer that can perform both BPE and unigram subword tokenization. Another advantage of subword tokenization is that it is language-independent and allows one to specify the size of the vocabulary. This helps to deal with out-of-vocabulary issues. First, let's train a SentencePiece model. SentencePiece classes are implemented in the [SIL.Machine.Tokenization.SentencePiece](https://www.nuget.org/packages/SIL.Machine.Tokenization.SentencePiece/) package.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - "using System.IO;\n", - "using SIL.Machine.Tokenization.SentencePiece;\n", - "\n", - "Directory.CreateDirectory(\"out\");\n", - "var trainer = new SentencePieceTrainer\n", - "{\n", - " VocabSize = 400,\n", - " ModelType = SentencePieceModelType.Unigram\n", - "};\n", - "trainer.Train(\"data/en.txt\", \"out/en-sp\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now that we have a SentencePiece model, we can split the text into subwords.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "▁Th | is | ▁ | is | ▁a | ▁ | t | es | t | .\n" - ] - } - ], - "source": [ - "{\n", - " using var tokenizer = new SentencePieceTokenizer(\"out/en-sp.model\");\n", - " var tokens = tokenizer.Tokenize(\"This is a test.\");\n", - " WriteLine(string.Join(\" | \", tokens));\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Detokenizing text\n", - "\n", - "For many NLP pipelines, tokens will need to be merged back into detokenized text. This is very common for machine translation. Many of the tokenizers in Machine also have a corresponding detokenizer that can be used to convert tokens back into a correct sequence. Once again, let's start with a simple, whitespace detokenizer.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This is a test .\n" - ] - } - ], - "source": [ - "var detokenizer = new WhitespaceDetokenizer();\n", - "var sentence = detokenizer.Detokenize(new[] { \"This\", \"is\", \"a\", \"test\", \".\" });\n", - "WriteLine(sentence);" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Machine has a general detokenizer that works well with languages with a Latin-based script.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\"This is a test, also.\"\n" - ] - } - ], - "source": [ - "var wordDetokenizer = new LatinWordDetokenizer();\n", - "var sentence = wordDetokenizer.Detokenize(new[] { \"\\\"\", \"This\", \"is\", \"a\", \"test\", \",\", \"also\", \".\", \"\\\"\" });\n", - "WriteLine(sentence);" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Machine has a detokenizer that properly deals with text that uses zero-width space to delimit words and spaces to delimit sentences.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Lorem​Ipsum​Dolor​Sit​Amet​Consectetur Adipiscing​Elit​Sed\n" - ] - } - ], - "source": [ - "var wordDetokenizer = new ZwspWordDetokenizer();\n", - "var sentence = wordDetokenizer.Detokenize(\n", - " new[] { \"Lorem\", \"Ipsum\", \"Dolor\", \"Sit\", \"Amet\", \"Consectetur\", \" \", \"Adipiscing\", \"Elit\", \"Sed\" });\n", - "WriteLine(sentence);" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Machine contains a detokenizer for SentencePiece encoded text. SentencePiece encodes spaces in the tokens, so that it can be detokenized without any ambiguities.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This is a test.\n" - ] - } - ], - "source": [ - "var detokenizer = new SentencePieceDetokenizer();\n", - "var sentence = detokenizer.Detokenize(new[] { \"▁Th\", \"is\", \"▁\", \"is\", \"▁a\", \"▁\", \"t\", \"es\", \"t\", \".\" });\n", - "WriteLine(sentence);" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".NET (C#)", - "language": "C#", - "name": ".net-csharp" - }, - "language_info": { - "name": "C#" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/samples/word_alignment.ipynb b/samples/word_alignment.ipynb deleted file mode 100644 index e0880eb0f..000000000 --- a/samples/word_alignment.ipynb +++ /dev/null @@ -1,435 +0,0 @@ -{ - "cells": [ - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Word Alignment Tutorial\n", - "\n", - "In this notebook, we will demonstrate how to use machine to train statistical word alignment models and then use them to predict alignments between sentences. Machine uses the [Thot](https://github.com/sillsdev/thot) library to implement word alignment models. The implementations are contained in the [SIL.Machine.Translation.Thot](https://www.nuget.org/packages/SIL.Machine.Translation.Thot/) package.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
Installed Packages
  • SIL.Scripture, 12.0.1
  • Thot, 3.4.1
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "#r \"nuget:SIL.Scripture,12.0.1\"\n", - "#r \"nuget:Thot\"\n", - "#r \"../src/SIL.Machine.Translation.Thot/bin/Debug/netstandard2.0/SIL.Machine.dll\"\n", - "#r \"../src/SIL.Machine.Translation.Thot/bin/Debug/netstandard2.0/SIL.Machine.Translation.Thot.dll\"\n", - "\n", - "void WriteLine(string text = \"\")\n", - "{\n", - " Console.Write(text + \"\\n\");\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Training models\n", - "\n", - "The first step in training a statistical word alignment model is setting up a parallel text corpus. Word alignment models are unsupervised, so they only require a parallel text corpus to train. Manually created alignments are not necessary. So let's create a parallel corpus from the source and target monolingual corpora.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [], - "source": [ - "using SIL.Machine.Corpora;\n", - "using SIL.Machine.Tokenization;\n", - "\n", - "var sourceCorpus = new ParatextTextCorpus(\"data/VBL-PT\");\n", - "var targetCorpus = new ParatextTextCorpus(\"data/WEB-PT\");\n", - "var parallelCorpus = sourceCorpus.AlignRows(targetCorpus).Tokenize();" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Machine has implementations of all common statistical models, including the famous IBM models (1-4), HMM, and FastAlign. All alignment models implement the `IWordAlignmentModel` interface. This makes it easier to swap out different models in your code.\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this tutorial, we are going to start by training an IBM-1 model. There are two possible ways to train a model. First, we will demonstrate training a model from a class that implements `IWordAlignmentModel`. We use the `CreateTrainer` method to create a trainer object that is used to train the model. If we do not specify a file path when creating the model object, then the model will only exist in memory. When we call the `SaveAsync` method, the model instance is updated with the trained model parameters, but the model is not written to disk. We are going to use the `Lowercase` token processor to pre-process that data, since that generally gives better results.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Training IBM-1 model: 0.00%\n", - "Training IBM-1 model: 16.67%\n", - "Training IBM-1 model: 33.33%\n", - "Training IBM-1 model: 50.00%\n", - "Training IBM-1 model: 66.67%\n", - "Training IBM-1 model: 83.33%\n", - "Training IBM-1 model: 100.00%\n" - ] - } - ], - "source": [ - "using SIL.Machine.Translation.Thot;\n", - "using SIL.Machine.Utils;\n", - "\n", - "{\n", - " using var model = new ThotIbm1WordAlignmentModel();\n", - " using var trainer = model.CreateTrainer(parallelCorpus.Lowercase());\n", - " await trainer.TrainAsync(new DelegateProgress(status => WriteLine($\"Training IBM-1 model: {status.PercentCompleted:P}\")));\n", - " await trainer.SaveAsync();\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The other option for training a model is to construct a trainer object directly. This method is useful for when you are only interested in training the model and saving it to disk for later use. We need to specify where the model will be saved after it is trained and we call the `SaveAsync` method.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Training IBM-1 model: 0.00%\n", - "Training IBM-1 model: 16.67%\n", - "Training IBM-1 model: 33.33%\n", - "Training IBM-1 model: 50.00%\n", - "Training IBM-1 model: 66.67%\n", - "Training IBM-1 model: 83.33%\n", - "Training IBM-1 model: 100.00%\n", - "IBM-1 model saved\n" - ] - } - ], - "source": [ - "using System.IO;\n", - "\n", - "Directory.CreateDirectory(\"out/VBL-WEB-IBM1\");\n", - "{\n", - " using var trainer = new ThotWordAlignmentModelTrainer(ThotWordAlignmentModelType.Ibm1, parallelCorpus.Lowercase(),\n", - " \"out/VBL-WEB-IBM1/src_trg\");\n", - " await trainer.TrainAsync(new DelegateProgress(status => WriteLine($\"Training IBM-1 model: {status.PercentCompleted:P}\")));\n", - " await trainer.SaveAsync();\n", - " WriteLine(\"IBM-1 model saved\");\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Aligning parallel sentences\n", - "\n", - "Now that we have a trained alignment model, we can find the best alignment for a parallel sentence. We call `Align` method to find the best alignment. The results are returned as a `WordAlignmentMatrix` object.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1JN 1:1\n", - "Source: Esta carta trata sobre la Palabra de vida que existía desde el principio , que hemos escuchado , que hemos visto con nuestros propios ojos y le hemos contemplado , y que hemos tocado con nuestras manos .\n", - "Target: That which was from the beginning , that which we have heard , that which we have seen with our eyes , that which we saw , and our hands touched , concerning the Word of life\n", - "Alignment: 2-2 2-20 2-25 2-29 2-30 3-32 4-4 4-33 5-34 6-35 7-36 12-3 12-5 13-6 13-12 13-21 13-26 13-31 15-0 15-7 15-10 15-13 15-16 15-22 16-1 16-8 16-11 16-14 16-23 20-17 21-18 22-19 22-28 25-27 35-9 35-15 35-24\n", - "1JN 1:2\n", - "Source: Esta Vida nos fue revelada . La vimos y damos testimonio de ella . Estamos hablándoles de Aquél que es la Vida Eterna , que estaba con el Padre , y que nos fue revelado .\n", - "Target: ( and the life was revealed , and we have seen , and testify , and declare to you the life , the eternal life , which was with the Father , and was revealed to us ) ;\n", - "Alignment: 0-17 0-35 1-3 1-20 1-24 2-36 3-4 3-27 3-33 4-0 4-10 4-13 4-16 4-37 8-1 8-7 8-12 8-15 8-32 10-26 12-18 14-8 22-23 23-6 23-11 23-14 23-21 23-25 23-31 26-28 28-2 28-9 28-19 28-22 28-29 28-30 34-5 34-34 34-38\n", - "1JN 1:3\n", - "Source: Los que hemos visto y oído eso mismo les contamos , para que también puedan participar de esta amistad junto a nosotros . Esta amistad con el Padre y su Hijo Jesucristo .\n", - "Target: that which we have seen and heard we declare to you , that you also may have fellowship with us . Yes , and our fellowship is with the Father and with his Son , Jesus Christ .\n", - "Alignment: 2-0 2-1 2-2 2-3 2-7 2-12 2-16 3-4 4-5 4-23 4-30 5-21 6-9 8-10 8-13 10-11 10-22 10-34 13-14 14-15 18-6 18-8 18-17 18-24 18-25 21-19 22-20 22-37 25-18 25-27 25-31 26-26 27-28 27-29 29-32 30-33 31-35 31-36\n", - "1JN 1:4\n", - "Source: Escribimos para decirles esto , a fin de que nuestra felicidad sea completa .\n", - "Target: And we write these things to you , that our joy may be fulfilled .\n", - "Alignment: 0-9 0-10 0-13 1-8 2-3 2-4 3-5 3-6 4-7 6-0 6-2 9-12 12-1 12-11 13-14\n", - "1JN 1:5\n", - "Source: Este es el mensaje que recibimos de él y que nosotros les declaramos a ustedes : Dios es luz , y no hay ningún vestigio de oscuridad en él .\n", - "Target: This is the message which we have heard from him and announce to you , that God is light , and in him is no darkness at all .\n", - "Alignment: 0-8 1-1 1-17 1-23 2-2 3-3 3-4 3-7 4-15 5-11 5-26 5-27 7-9 7-22 8-10 8-20 10-5 14-6 14-13 15-0 16-16 18-18 19-14 19-19 22-12 22-24 26-25 27-21 29-28\n" - ] - } - ], - "source": [ - "{\n", - " using var model = new ThotIbm1WordAlignmentModel(\"out/VBL-WEB-IBM1/src_trg\");\n", - " foreach (var row in parallelCorpus.Take(5))\n", - " {\n", - " var alignment = model.Align(row.SourceSegment.Lowercase(), row.TargetSegment.Lowercase());\n", - "\n", - " WriteLine($\"{row.Ref}\");\n", - " WriteLine($\"Source: {row.SourceText}\");\n", - " WriteLine($\"Target: {row.TargetText}\");\n", - " WriteLine($\"Alignment: {alignment}\");\n", - " }\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Getting model probabilities\n", - "\n", - "A statistical word alignment model consists of one or more conditional probability distributions that are estimated from the training data. For example, most models estimate a word translation probability distribution that can be queried to obtain the probability that a source word is a translation of a target word. Each model class has methods to obtain these probabilities. Let's try getting some translation probabilities from the IBM-1 model that we trained by calling the `GetTranslationProbability` method. In order to get the probability that a word does not translate to anything, you can pass `null` instead of the word string.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "es -> is: 0.2720\n", - "NULL -> that: 0.0516\n" - ] - } - ], - "source": [ - "{\n", - " using var model = new ThotIbm1WordAlignmentModel(\"out/VBL-WEB-IBM1/src_trg\");\n", - " double prob = model.GetTranslationProbability(\"es\", \"is\");\n", - " WriteLine($\"es -> is: {prob:0.0000}\");\n", - " prob = model.GetTranslationProbability(null, \"that\");\n", - " WriteLine($\"NULL -> that: {prob:0.0000}\");\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Symmetrized alignment models\n", - "\n", - "Most statistical word alignment models are directional and asymmetric. This means that it can only model one-to-one and one-to-many alignments in one direction. They are not capable of modeling many-to-many alignments, which can occur in some language pairs. One way to get around this limitation is to train models in both directions (source-to-target and target-to-source), and then merge the resulting alignments from the two models into a single alignment. This is called symmetrization and is a common practice when using statistical word alignment models. In addition, researchers have found that symmetrized alignments are better quality.\n" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Machine provides a special word alignment model class to support symmetrization called `SymmetrizedWordAlignmentModel`. Let's demonstrate how to use this class. First, we will train the symmetrized model using the `SymmetrizedWordAlignmentModelTrainer` class.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Training direct alignment model: 0.00%\n", - "Training direct alignment model: 0.00%\n", - "Training direct alignment model: 8.33%\n", - "Training direct alignment model: 16.67%\n", - "Training direct alignment model: 25.00%\n", - "Training direct alignment model: 33.33%\n", - "Training direct alignment model: 41.67%\n", - "Training direct alignment model: 50.00%\n", - "Training inverse alignment model: 50.00%\n", - "Training inverse alignment model: 50.00%\n", - "Training inverse alignment model: 58.33%\n", - "Training inverse alignment model: 66.67%\n", - "Training inverse alignment model: 75.00%\n", - "Training inverse alignment model: 83.33%\n", - "Training inverse alignment model: 91.67%\n", - "Training inverse alignment model: 100.00%\n", - "Symmetrized IBM-1 model saved\n" - ] - } - ], - "source": [ - "using SIL.Machine.Translation;\n", - "\n", - "{\n", - " using var srcTrgTrainer = new ThotWordAlignmentModelTrainer(ThotWordAlignmentModelType.Ibm1,\n", - " parallelCorpus.Lowercase(), \"out/VBL-WEB-IBM1/src_trg\");\n", - " using var trgSrcTrainer = new ThotWordAlignmentModelTrainer(ThotWordAlignmentModelType.Ibm1,\n", - " parallelCorpus.Invert().Lowercase(), \"out/VBL-WEB-IBM1/trg_src\");\n", - " using var symmetrizedTrainer = new SymmetrizedWordAlignmentModelTrainer(srcTrgTrainer, trgSrcTrainer);\n", - " await symmetrizedTrainer.TrainAsync(new DelegateProgress(status =>\n", - " WriteLine($\"{status.Message}: {status.PercentCompleted:P}\")));\n", - " await symmetrizedTrainer.SaveAsync();\n", - " WriteLine(\"Symmetrized IBM-1 model saved\");\n", - "}" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The model can also be trained using the `CreateTrainer` method on `SymmetrizedWordAlignmentModel`. Now that we've trained the symmetrized model, let's obtain some alignments. Machine supports many different symmetrization heuristics. The symmetrization heuristic to use when merging alignments can be specified using the `Heuristic` property. In this case, we will use the `GrowDiagFinalAnd` heuristic.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "dotnet_interactive": { - "language": "csharp" - }, - "vscode": { - "languageId": "polyglot-notebook" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1JN 1:1\n", - "Source: Esta carta trata sobre la Palabra de vida que existía desde el principio , que hemos escuchado , que hemos visto con nuestros propios ojos y le hemos contemplado , y que hemos tocado con nuestras manos .\n", - "Target: That which was from the beginning , that which we have heard , that which we have seen with our eyes , that which we saw , and our hands touched , concerning the Word of life\n", - "Alignment: 1-25 2-25 3-32 4-4 4-33 5-34 6-35 7-36 8-0 12-5 13-6 15-10 16-11 19-20 20-17 21-18 22-19 25-27 35-9\n", - "1JN 1:2\n", - "Source: Esta Vida nos fue revelada . La vimos y damos testimonio de ella . Estamos hablándoles de Aquél que es la Vida Eterna , que estaba con el Padre , y que nos fue revelado .\n", - "Target: ( and the life was revealed , and we have seen , and testify , and declare to you the life , the eternal life , which was with the Father , and was revealed to us ) ;\n", - "Alignment: 0-17 1-3 2-36 3-4 4-0 6-38 7-0 8-1 9-0 10-0 11-13 12-18 14-8 18-9 22-23 23-6 26-28 27-2 28-29 28-30 33-4 34-5\n", - "1JN 1:3\n", - "Source: Los que hemos visto y oído eso mismo les contamos , para que también puedan participar de esta amistad junto a nosotros . Esta amistad con el Padre y su Hijo Jesucristo .\n", - "Target: that which we have seen and heard we declare to you , that you also may have fellowship with us . Yes , and our fellowship is with the Father and with his Son , Jesus Christ .\n", - "Alignment: 1-0 2-1 2-2 2-3 3-4 4-5 5-21 6-9 8-10 10-11 13-14 14-15 17-8 18-17 19-17 21-19 22-20 24-17 25-18 26-26 27-28 27-29 29-32 30-33 31-35 31-36\n", - "1JN 1:4\n", - "Source: Escribimos para decirles esto , a fin de que nuestra felicidad sea completa .\n", - "Target: And we write these things to you , that our joy may be fulfilled .\n", - "Alignment: 0-13 1-8 2-4 3-3 3-5 3-6 4-7 6-0 9-12 10-13 11-13 12-13 13-14\n", - "1JN 1:5\n", - "Source: Este es el mensaje que recibimos de él y que nosotros les declaramos a ustedes : Dios es luz , y no hay ningún vestigio de oscuridad en él .\n", - "Target: This is the message which we have heard from him and announce to you , that God is light , and in him is no darkness at all .\n", - "Alignment: 0-8 1-0 1-1 2-2 3-3 3-4 4-15 5-11 6-27 7-9 8-10 10-5 12-11 13-12 14-13 16-16 18-18 19-14 19-19 21-24 22-24 26-25 27-21 29-28\n" - ] - } - ], - "source": [ - "{\n", - " using var srcTrgModel = new ThotIbm1WordAlignmentModel(\"out/VBL-WEB-IBM1/src_trg\");\n", - " using var trgSrcModel = new ThotIbm1WordAlignmentModel(\"out/VBL-WEB-IBM1/trg_src\");\n", - " using var symmetrizedModel = new SymmetrizedWordAlignmentModel(srcTrgModel, trgSrcModel)\n", - " {\n", - " Heuristic = SymmetrizationHeuristic.GrowDiagFinalAnd\n", - " };\n", - " foreach (var row in parallelCorpus.Take(5))\n", - " {\n", - " var alignment = symmetrizedModel.Align(row.SourceSegment.Lowercase(), row.TargetSegment.Lowercase());\n", - "\n", - " WriteLine($\"{row.Ref}\");\n", - " WriteLine($\"Source: {row.SourceText}\");\n", - " WriteLine($\"Target: {row.TargetText}\");\n", - " WriteLine($\"Alignment: {alignment}\");\n", - " }\n", - "}" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".NET (C#)", - "language": "C#", - "name": ".net-csharp" - }, - "language_info": { - "name": "C#" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/SIL.Machine.AspNetCore/Configuration/BuildJobOptions.cs b/src/SIL.Machine.AspNetCore/Configuration/BuildJobOptions.cs deleted file mode 100644 index b6dc8c328..000000000 --- a/src/SIL.Machine.AspNetCore/Configuration/BuildJobOptions.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace SIL.Machine.AspNetCore.Configuration; - -public class BuildJobOptions -{ - public const string Key = "BuildJob"; - - public IList ClearML { get; set; } = new List(); -} diff --git a/src/SIL.Machine.AspNetCore/Configuration/ClearMLBuildQueue.cs b/src/SIL.Machine.AspNetCore/Configuration/ClearMLBuildQueue.cs deleted file mode 100644 index 9ffe279d8..000000000 --- a/src/SIL.Machine.AspNetCore/Configuration/ClearMLBuildQueue.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace SIL.Machine.AspNetCore.Configuration; - -public class ClearMLBuildQueue -{ - public TranslationEngineType TranslationEngineType { get; set; } - public string ModelType { get; set; } = ""; - public string Queue { get; set; } = "default"; - public string DockerImage { get; set; } = ""; -} diff --git a/src/SIL.Machine.AspNetCore/Configuration/ClearMLOptions.cs b/src/SIL.Machine.AspNetCore/Configuration/ClearMLOptions.cs deleted file mode 100644 index fdc9e0238..000000000 --- a/src/SIL.Machine.AspNetCore/Configuration/ClearMLOptions.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace SIL.Machine.AspNetCore.Configuration; - -public class ClearMLOptions -{ - public const string Key = "ClearML"; - - public string AccessKey { get; set; } = ""; - public string SecretKey { get; set; } = ""; - public bool BuildPollingEnabled { get; set; } = false; - public TimeSpan BuildPollingTimeout { get; set; } = TimeSpan.FromSeconds(10); - public string RootProject { get; set; } = "Machine"; - public string Project { get; set; } = "dev"; -} diff --git a/src/SIL.Machine.AspNetCore/Configuration/IEndpointRouteBuilderExtensions.cs b/src/SIL.Machine.AspNetCore/Configuration/IEndpointRouteBuilderExtensions.cs deleted file mode 100644 index 694dd67eb..000000000 --- a/src/SIL.Machine.AspNetCore/Configuration/IEndpointRouteBuilderExtensions.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Microsoft.AspNetCore.Builder; - -public static class IEndpointRouteBuilderExtensions -{ - public static IEndpointRouteBuilder MapServalTranslationEngineService(this IEndpointRouteBuilder builder) - { - builder.MapGrpcService(); - - return builder; - } -} diff --git a/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilder.cs b/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilder.cs deleted file mode 100644 index f8dfbcd5a..000000000 --- a/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilder.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public interface IMachineBuilder -{ - IServiceCollection Services { get; } - IConfiguration? Configuration { get; } -} diff --git a/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs b/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs deleted file mode 100644 index 3d10565f6..000000000 --- a/src/SIL.Machine.AspNetCore/Configuration/IMachineBuilderExtensions.cs +++ /dev/null @@ -1,437 +0,0 @@ -using Serval.Translation.V1; - -namespace Microsoft.Extensions.DependencyInjection; - -public static class IMachineBuilderExtensions -{ - public static IMachineBuilder AddServiceOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddServiceOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSmtTransferEngineOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddClearMLOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddClearMLOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddMessageOutboxOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddMessageOutboxOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddSharedFileOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddBuildJobOptions( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - return builder; - } - - public static IMachineBuilder AddBuildJobOptions(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - return builder; - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder) - { - if (builder.Configuration is null) - return builder.AddThotSmtModel(o => { }); - else - return builder.AddThotSmtModel(builder.Configuration.GetSection(ThotSmtModelOptions.Key)); - } - - public static IMachineBuilder AddThotSmtModel( - this IMachineBuilder builder, - Action configureOptions - ) - { - builder.Services.Configure(configureOptions); - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddThotSmtModel(this IMachineBuilder builder, IConfiguration config) - { - builder.Services.Configure(config); - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddTransferEngine(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddUnigramTruecaser(this IMachineBuilder builder) - { - builder.Services.AddSingleton(); - return builder; - } - - public static IMachineBuilder AddClearMLService(this IMachineBuilder builder, string? connectionString = null) - { - connectionString ??= builder.Configuration?.GetConnectionString("ClearML"); - if (connectionString is null) - throw new InvalidOperationException("ClearML connection string is required"); - - builder - .Services.AddHttpClient("ClearML") - .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!)) - // Add retry policy; fail after approx. 2 + 4 + 8 = 14 seconds - .AddTransientHttpErrorPolicy(b => - b.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))) - ); - - builder.Services.AddSingleton(); - - // workaround register satisfying the interface and as a hosted service. - builder.Services.AddSingleton(); - builder.Services.AddHostedService(p => p.GetRequiredService()); - - builder - .Services.AddHttpClient("ClearML-NoRetry") - .ConfigureHttpClient(httpClient => httpClient.BaseAddress = new Uri(connectionString!)); - builder.Services.AddSingleton(); - - builder.Services.AddHealthChecks().AddCheck("ClearML Health Check"); - - return builder; - } - - private static MongoStorageOptions GetMongoStorageOptions() - { - var mongoStorageOptions = new MongoStorageOptions - { - MigrationOptions = new MongoMigrationOptions - { - MigrationStrategy = new MigrateMongoMigrationStrategy(), - BackupStrategy = new CollectionMongoBackupStrategy() - }, - CheckConnection = true, - CheckQueuedJobsStrategy = CheckQueuedJobsStrategy.TailNotificationsCollection, - }; - return mongoStorageOptions; - } - - public static IMachineBuilder AddMongoHangfireJobClient( - this IMachineBuilder builder, - string? connectionString = null - ) - { - connectionString ??= builder.Configuration?.GetConnectionString("Hangfire"); - if (connectionString is null) - throw new InvalidOperationException("Hangfire connection string is required"); - - builder.Services.AddHangfire(c => - c.SetDataCompatibilityLevel(CompatibilityLevel.Version_170) - .UseSimpleAssemblyNameTypeSerializer() - .UseRecommendedSerializerSettings() - .UseMongoStorage(connectionString, GetMongoStorageOptions()) - .UseFilter(new AutomaticRetryAttribute { Attempts = 0 }) - ); - builder.Services.AddHealthChecks().AddCheck(name: "Hangfire"); - return builder; - } - - public static IMachineBuilder AddHangfireJobServer( - this IMachineBuilder builder, - IEnumerable? engineTypes = null - ) - { - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? [TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt]; - var queues = new List(); - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.Services.AddSingleton(); - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - queues.Add("smt_transfer"); - break; - case TranslationEngineType.Nmt: - queues.Add("nmt"); - break; - } - } - - builder.Services.AddHangfireServer(o => - { - o.Queues = queues.ToArray(); - }); - return builder; - } - - public static IMachineBuilder AddMemoryDataAccess(this IMachineBuilder builder) - { - builder.Services.AddMemoryDataAccess(o => - { - o.AddRepository(); - o.AddRepository(); - o.AddRepository(); - o.AddRepository(); - o.AddRepository(); - }); - - return builder; - } - - public static IMachineBuilder AddMongoDataAccess(this IMachineBuilder builder, string? connectionString = null) - { - connectionString ??= builder.Configuration?.GetConnectionString("Mongo"); - if (connectionString is null) - throw new InvalidOperationException("Mongo connection string is required"); - builder.Services.AddMongoDataAccess( - connectionString!, - "SIL.Machine.AspNetCore.Models", - o => - { - o.AddRepository( - "translation_engines", - mapSetup: m => m.SetIgnoreExtraElements(true), - init: async c => - { - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders - .IndexKeys.Ascending(e => e.EngineId) - .Ascending("currentBuild._id") - ) - ); - await c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(e => e.CurrentBuild!.BuildJobRunner) - ) - ); - } - ); - o.AddRepository("locks"); - o.AddRepository( - "train_segment_pairs", - init: c => - c.Indexes.CreateOrUpdateAsync( - new CreateIndexModel( - Builders.IndexKeys.Ascending(p => p.TranslationEngineRef) - ) - ) - ); - o.AddRepository( - "outbox_messages", - mapSetup: m => m.MapProperty(m => m.OutboxRef).SetSerializer(new StringSerializer()) - ); - o.AddRepository( - "outboxes", - mapSetup: m => m.MapIdProperty(o => o.Id).SetSerializer(new StringSerializer()) - ); - } - ); - builder.Services.AddHealthChecks().AddMongoDb(connectionString!, name: "Mongo"); - - return builder; - } - - public static IMachineBuilder AddServalPlatformService( - this IMachineBuilder builder, - string? connectionString = null - ) - { - connectionString ??= builder.Configuration?.GetConnectionString("Serval"); - if (connectionString is null) - throw new InvalidOperationException("Serval connection string is required"); - - builder.Services.AddScoped(); - - builder.Services.AddSingleton(); - - builder.Services.AddScoped(); - - builder - .Services.AddGrpcClient(o => - { - o.Address = new Uri(connectionString); - }) - .ConfigureChannel(o => - { - o.MaxRetryAttempts = null; - o.ServiceConfig = new ServiceConfig - { - MethodConfigs = - { - new MethodConfig - { - Names = { MethodName.Default }, - RetryPolicy = new Grpc.Net.Client.Configuration.RetryPolicy - { - MaxAttempts = 10, - InitialBackoff = TimeSpan.FromSeconds(1), - MaxBackoff = TimeSpan.FromSeconds(5), - BackoffMultiplier = 1.5, - RetryableStatusCodes = { StatusCode.Unavailable } - } - }, - new MethodConfig - { - Names = - { - new MethodName - { - Service = "serval.translation.v1.TranslationPlatformApi", - Method = "UpdateBuildStatus" - } - } - }, - } - }; - }); - - return builder; - } - - public static IMachineBuilder AddServalTranslationEngineService( - this IMachineBuilder builder, - string? connectionString = null, - IEnumerable? engineTypes = null - ) - { - builder.Services.AddGrpc(options => - { - options.Interceptors.Add(); - options.Interceptors.Add(); - }); - builder.AddServalPlatformService(connectionString); - - engineTypes ??= - builder.Configuration?.GetSection("TranslationEngines").Get() - ?? [TranslationEngineType.SmtTransfer, TranslationEngineType.Nmt]; - foreach (TranslationEngineType engineType in engineTypes.Distinct()) - { - switch (engineType) - { - case TranslationEngineType.SmtTransfer: - builder.Services.AddSingleton(); - builder.Services.AddHostedService(); - builder.AddThotSmtModel().AddTransferEngine().AddUnigramTruecaser(); - builder.Services.AddScoped(); - break; - case TranslationEngineType.Nmt: - builder.Services.AddScoped(); - break; - } - } - - return builder; - } - - public static IMachineBuilder AddBuildJobService(this IMachineBuilder builder, string? smtTransferEngineDir = null) - { - builder.Services.AddScoped(); - - builder.Services.AddScoped(); - builder.Services.AddScoped(); - builder.Services.AddScoped(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(x => x.GetRequiredService()); - builder.Services.AddHostedService(p => p.GetRequiredService()); - - builder.Services.AddScoped(); - builder.Services.AddScoped(); - builder.Services.AddScoped(); - - if (smtTransferEngineDir is null) - { - var smtTransferEngineOptions = new SmtTransferEngineOptions(); - builder.Configuration?.GetSection(SmtTransferEngineOptions.Key).Bind(smtTransferEngineOptions); - smtTransferEngineDir = smtTransferEngineOptions.EnginesDir; - } - string? driveLetter = Path.GetPathRoot(smtTransferEngineDir)?[..1]; - if (driveLetter is null) - throw new InvalidOperationException("SMT Engine directory is required"); - // add health check for disk storage capacity - builder - .Services.AddHealthChecks() - .AddDiskStorageHealthCheck( - x => x.AddDrive(driveLetter, 1_000), // 1GB - "SMT Engine Storage Capacity", - HealthStatus.Degraded - ); - - return builder; - } - - public static IMachineBuilder AddModelCleanupService(this IMachineBuilder builder) - { - builder.Services.AddHostedService(); - return builder; - } - - public static IMachineBuilder AddMessageOutboxDeliveryService(this IMachineBuilder builder) - { - builder.Services.AddHostedService(); - return builder; - } -} diff --git a/src/SIL.Machine.AspNetCore/Configuration/IServiceCollectionExtensions.cs b/src/SIL.Machine.AspNetCore/Configuration/IServiceCollectionExtensions.cs deleted file mode 100644 index 7463e6acd..000000000 --- a/src/SIL.Machine.AspNetCore/Configuration/IServiceCollectionExtensions.cs +++ /dev/null @@ -1,54 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -public static class IServiceCollectionExtensions -{ - public static IMachineBuilder AddMachine(this IServiceCollection services, IConfiguration? configuration = null) - { - if (!Sldr.IsInitialized) - Sldr.Initialize(); - - services.AddSingleton(); - services.AddSingleton(); - services.AddHealthChecks().AddCheck("S3 Bucket"); - - services.AddSingleton(); - services.AddTransient(); - - services.AddScoped(); - services.AddSingleton(); - services.AddStartupTask( - (sp, cancellationToken) => - sp.GetRequiredService().InitAsync(cancellationToken) - ); - - var builder = new MachineBuilder(services, configuration); - if (configuration is null) - { - builder.AddServiceOptions(o => { }); - builder.AddSharedFileOptions(o => { }); - builder.AddSmtTransferEngineOptions(o => { }); - builder.AddClearMLOptions(o => { }); - builder.AddBuildJobOptions(o => { }); - builder.AddMessageOutboxOptions(o => { }); - } - else - { - builder.AddServiceOptions(configuration.GetSection(ServiceOptions.Key)); - builder.AddSharedFileOptions(configuration.GetSection(SharedFileOptions.Key)); - builder.AddSmtTransferEngineOptions(configuration.GetSection(SmtTransferEngineOptions.Key)); - builder.AddClearMLOptions(configuration.GetSection(ClearMLOptions.Key)); - builder.AddBuildJobOptions(configuration.GetSection(BuildJobOptions.Key)); - builder.AddMessageOutboxOptions(configuration.GetSection(MessageOutboxOptions.Key)); - } - return builder; - } - - public static IServiceCollection AddStartupTask( - this IServiceCollection services, - Func startupTask - ) - { - services.AddHostedService(sp => new StartupTask(sp, startupTask)); - return services; - } -} diff --git a/src/SIL.Machine.AspNetCore/Configuration/MachineBuilder.cs b/src/SIL.Machine.AspNetCore/Configuration/MachineBuilder.cs deleted file mode 100644 index 58ddf5c15..000000000 --- a/src/SIL.Machine.AspNetCore/Configuration/MachineBuilder.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Microsoft.Extensions.DependencyInjection; - -internal class MachineBuilder(IServiceCollection services, IConfiguration? configuration) : IMachineBuilder -{ - public IServiceCollection Services { get; } = services; - public IConfiguration? Configuration { get; } = configuration; -} diff --git a/src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs b/src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs deleted file mode 100644 index e4b9946bf..000000000 --- a/src/SIL.Machine.AspNetCore/Configuration/MessageOutboxOptions.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace SIL.Machine.AspNetCore.Configuration; - -public class MessageOutboxOptions -{ - public const string Key = "MessageOutbox"; - - public string OutboxDir { get; set; } = "outbox"; - public TimeSpan MessageExpirationTimeout { get; set; } = TimeSpan.FromHours(48); -} diff --git a/src/SIL.Machine.AspNetCore/Configuration/ServiceOptions.cs b/src/SIL.Machine.AspNetCore/Configuration/ServiceOptions.cs deleted file mode 100644 index 37bc17d9d..000000000 --- a/src/SIL.Machine.AspNetCore/Configuration/ServiceOptions.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace SIL.Machine.AspNetCore.Configuration; - -public class ServiceOptions -{ - public const string Key = "Service"; - - public string ServiceId { get; set; } = "machine_api"; -} diff --git a/src/SIL.Machine.AspNetCore/Configuration/SharedFileOptions.cs b/src/SIL.Machine.AspNetCore/Configuration/SharedFileOptions.cs deleted file mode 100644 index a732296f9..000000000 --- a/src/SIL.Machine.AspNetCore/Configuration/SharedFileOptions.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace SIL.Machine.AspNetCore.Configuration; - -public class SharedFileOptions -{ - public const string Key = "SharedFile"; - - public string Uri { get; set; } = "file:///var/lib/machine/"; - public string S3AccessKeyId { get; set; } = ""; - public string S3SecretAccessKey { get; set; } = ""; - public string S3Region { get; set; } = "us-east-1"; -} diff --git a/src/SIL.Machine.AspNetCore/Configuration/SmtTransferEngineOptions.cs b/src/SIL.Machine.AspNetCore/Configuration/SmtTransferEngineOptions.cs deleted file mode 100644 index 67df3d1d5..000000000 --- a/src/SIL.Machine.AspNetCore/Configuration/SmtTransferEngineOptions.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace SIL.Machine.AspNetCore.Configuration; - -public class SmtTransferEngineOptions -{ - public const string Key = "SmtTransferEngine"; - - public string EnginesDir { get; set; } = "translation_engines"; - public TimeSpan EngineCommitFrequency { get; set; } = TimeSpan.FromMinutes(5); - public TimeSpan InactiveEngineTimeout { get; set; } = TimeSpan.FromMinutes(10); -} diff --git a/src/SIL.Machine.AspNetCore/Configuration/ThotSmtModelOptions.cs b/src/SIL.Machine.AspNetCore/Configuration/ThotSmtModelOptions.cs deleted file mode 100644 index 5941cac46..000000000 --- a/src/SIL.Machine.AspNetCore/Configuration/ThotSmtModelOptions.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace SIL.Machine.AspNetCore.Configuration; - -public class ThotSmtModelOptions -{ - public const string Key = "ThotSmtModel"; - - public ThotSmtModelOptions() - { - string installDir = Path.GetDirectoryName(Assembly.GetEntryAssembly()!.Location)!; - NewModelFile = Path.Combine(installDir, "thot-new-model.zip"); - } - - public string NewModelFile { get; set; } -} diff --git a/src/SIL.Machine.AspNetCore/Models/Build.cs b/src/SIL.Machine.AspNetCore/Models/Build.cs deleted file mode 100644 index 89a4bdc83..000000000 --- a/src/SIL.Machine.AspNetCore/Models/Build.cs +++ /dev/null @@ -1,32 +0,0 @@ -namespace SIL.Machine.AspNetCore.Models; - -public enum BuildJobState -{ - None, - Pending, - Active, - Canceling -} - -public enum BuildJobRunnerType -{ - Hangfire, - ClearML -} - -public enum BuildStage -{ - Preprocess, - Train, - Postprocess -} - -public record Build -{ - public required string BuildId { get; init; } - public required BuildJobState JobState { get; init; } - public required string JobId { get; init; } - public required BuildJobRunnerType BuildJobRunner { get; init; } - public required BuildStage Stage { get; init; } - public string? Options { get; set; } -} diff --git a/src/SIL.Machine.AspNetCore/Models/ClearMLMetricsEvent.cs b/src/SIL.Machine.AspNetCore/Models/ClearMLMetricsEvent.cs deleted file mode 100644 index 69d56946d..000000000 --- a/src/SIL.Machine.AspNetCore/Models/ClearMLMetricsEvent.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace SIL.Machine.AspNetCore.Models; - -public record ClearMLMetricsEvent -{ - public string? Metric { get; init; } - public string? Variant { get; init; } - public required double Value { get; init; } - public double? MinValue { get; init; } - public int? MinValueIteration { get; init; } - public double? MaxValue { get; init; } - public int? MaxValueIteration { get; init; } -} diff --git a/src/SIL.Machine.AspNetCore/Models/ClearMLProject.cs b/src/SIL.Machine.AspNetCore/Models/ClearMLProject.cs deleted file mode 100644 index ca6cf6883..000000000 --- a/src/SIL.Machine.AspNetCore/Models/ClearMLProject.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace SIL.Machine.AspNetCore.Models; - -public record ClearMLProject -{ - public required string Id { get; init; } -} diff --git a/src/SIL.Machine.AspNetCore/Models/ClearMLTask.cs b/src/SIL.Machine.AspNetCore/Models/ClearMLTask.cs deleted file mode 100644 index 9a05325ce..000000000 --- a/src/SIL.Machine.AspNetCore/Models/ClearMLTask.cs +++ /dev/null @@ -1,35 +0,0 @@ -namespace SIL.Machine.AspNetCore.Models; - -public enum ClearMLTaskStatus -{ - Created, - Queued, - InProgress, - Stopped, - Published, - Publishing, - Closed, - Failed, - Completed, - Unknown -} - -public record ClearMLTask -{ - public required string Id { get; init; } - public required string Name { get; init; } - public required ClearMLProject Project { get; init; } - public required ClearMLTaskStatus Status { get; init; } - public string? StatusReason { get; init; } - public string? StatusMessage { get; init; } - public required DateTime Created { get; init; } - public int? LastIteration { get; init; } - public int ActiveDuration { get; init; } - public required IReadOnlyDictionary< - string, - IReadOnlyDictionary - > LastMetrics { get; init; } - - [JsonConverter(typeof(DictionaryStringStringConverter))] - public required IReadOnlyDictionary Runtime { get; init; } -} diff --git a/src/SIL.Machine.AspNetCore/Models/Corpus.cs b/src/SIL.Machine.AspNetCore/Models/Corpus.cs deleted file mode 100644 index a6847555d..000000000 --- a/src/SIL.Machine.AspNetCore/Models/Corpus.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace SIL.Machine.AspNetCore.Models; - -public record Corpus -{ - public required string Id { get; init; } - public required string SourceLanguage { get; init; } - public required string TargetLanguage { get; init; } - public IReadOnlyDictionary>? TrainOnChapters { get; init; } - public IReadOnlyDictionary>? PretranslateChapters { get; init; } - public required HashSet? TrainOnTextIds { get; init; } - public required HashSet? PretranslateTextIds { get; init; } - public required IReadOnlyList SourceFiles { get; init; } - public required IReadOnlyList TargetFiles { get; init; } -} diff --git a/src/SIL.Machine.AspNetCore/Models/CorpusFile.cs b/src/SIL.Machine.AspNetCore/Models/CorpusFile.cs deleted file mode 100644 index 4df8dd8c1..000000000 --- a/src/SIL.Machine.AspNetCore/Models/CorpusFile.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace SIL.Machine.AspNetCore.Models; - -public enum FileFormat -{ - Text = 0, - Paratext = 1 -} - -public record CorpusFile -{ - public required string Location { get; init; } - public required FileFormat Format { get; init; } - public required string TextId { get; init; } -} diff --git a/src/SIL.Machine.AspNetCore/Models/Lock.cs b/src/SIL.Machine.AspNetCore/Models/Lock.cs deleted file mode 100644 index 3ea408614..000000000 --- a/src/SIL.Machine.AspNetCore/Models/Lock.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace SIL.Machine.AspNetCore.Models; - -public record Lock -{ - public required string Id { get; init; } - public DateTime? ExpiresAt { get; init; } - public required string HostId { get; init; } -} diff --git a/src/SIL.Machine.AspNetCore/Models/ModelDownloadUrl.cs b/src/SIL.Machine.AspNetCore/Models/ModelDownloadUrl.cs deleted file mode 100644 index a00da8f57..000000000 --- a/src/SIL.Machine.AspNetCore/Models/ModelDownloadUrl.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace SIL.Machine.AspNetCore.Models; - -public record ModelDownloadUrl -{ - public required string Url { get; init; } - public required int ModelRevision { get; init; } - public required DateTime ExpiresAt { get; init; } -} diff --git a/src/SIL.Machine.AspNetCore/Models/Outbox.cs b/src/SIL.Machine.AspNetCore/Models/Outbox.cs deleted file mode 100644 index 63e9441e4..000000000 --- a/src/SIL.Machine.AspNetCore/Models/Outbox.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace SIL.Machine.AspNetCore.Models; - -public record Outbox : IEntity -{ - public string Id { get; set; } = ""; - - public int Revision { get; set; } - - public int CurrentIndex { get; init; } -} diff --git a/src/SIL.Machine.AspNetCore/Models/OutboxMessage.cs b/src/SIL.Machine.AspNetCore/Models/OutboxMessage.cs deleted file mode 100644 index 8ec9462cd..000000000 --- a/src/SIL.Machine.AspNetCore/Models/OutboxMessage.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace SIL.Machine.AspNetCore.Models; - -public record OutboxMessage : IEntity -{ - public string Id { get; set; } = ""; - public int Revision { get; set; } = 1; - public required int Index { get; init; } - public required string OutboxRef { get; init; } - public required string Method { get; init; } - public required string GroupId { get; init; } - public string? Content { get; init; } - public required bool HasContentStream { get; init; } - public DateTimeOffset Created { get; init; } = DateTimeOffset.UtcNow; - public int Attempts { get; init; } -} diff --git a/src/SIL.Machine.AspNetCore/Models/Pretranslation.cs b/src/SIL.Machine.AspNetCore/Models/Pretranslation.cs deleted file mode 100644 index 31d895184..000000000 --- a/src/SIL.Machine.AspNetCore/Models/Pretranslation.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace SIL.Machine.AspNetCore.Models; - -public record Pretranslation -{ - public required string CorpusId { get; init; } - public required string TextId { get; init; } - public required IReadOnlyList Refs { get; init; } - public required string Translation { get; init; } -} diff --git a/src/SIL.Machine.AspNetCore/Models/RWLock.cs b/src/SIL.Machine.AspNetCore/Models/RWLock.cs deleted file mode 100644 index 6475e29f5..000000000 --- a/src/SIL.Machine.AspNetCore/Models/RWLock.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace SIL.Machine.AspNetCore.Models; - -public record RWLock : IEntity -{ - public string Id { get; set; } = ""; - public int Revision { get; set; } = 1; - public Lock? WriterLock { get; init; } - public required IReadOnlyList ReaderLocks { get; init; } - public required IReadOnlyList WriterQueue { get; init; } - - public bool IsAvailableForReading() - { - var now = DateTime.UtcNow; - return (WriterLock is null || WriterLock.ExpiresAt is not null && WriterLock.ExpiresAt <= now) - && WriterQueue.Count == 0; - } - - public bool IsAvailableForWriting(string? lockId = null) - { - var now = DateTime.UtcNow; - return (WriterLock is null || WriterLock.ExpiresAt is not null && WriterLock.ExpiresAt <= now) - && !ReaderLocks.Any(l => l.ExpiresAt is null || l.ExpiresAt > now) - && (lockId is null || WriterQueue.FirstOrDefault()?.Id == lockId); - } -} diff --git a/src/SIL.Machine.AspNetCore/Models/TrainSegmentPair.cs b/src/SIL.Machine.AspNetCore/Models/TrainSegmentPair.cs deleted file mode 100644 index 471b5296d..000000000 --- a/src/SIL.Machine.AspNetCore/Models/TrainSegmentPair.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace SIL.Machine.AspNetCore.Models; - -public record TrainSegmentPair : IEntity -{ - public string Id { get; set; } = ""; - public int Revision { get; set; } = 1; - public required string TranslationEngineRef { get; init; } - public required string Source { get; init; } - public required string Target { get; init; } - public required bool SentenceStart { get; init; } -} diff --git a/src/SIL.Machine.AspNetCore/Models/TranslationEngine.cs b/src/SIL.Machine.AspNetCore/Models/TranslationEngine.cs deleted file mode 100644 index cedb504c6..000000000 --- a/src/SIL.Machine.AspNetCore/Models/TranslationEngine.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace SIL.Machine.AspNetCore.Models; - -public record TranslationEngine : IEntity -{ - public string Id { get; set; } = ""; - public int Revision { get; set; } = 1; - public required string EngineId { get; init; } - public required TranslationEngineType Type { get; init; } - public required string SourceLanguage { get; init; } - public required string TargetLanguage { get; init; } - public required bool IsModelPersisted { get; init; } - public int BuildRevision { get; init; } - public Build? CurrentBuild { get; init; } -} diff --git a/src/SIL.Machine.AspNetCore/Properties/AssemblyInfo.cs b/src/SIL.Machine.AspNetCore/Properties/AssemblyInfo.cs deleted file mode 100644 index a6d427972..000000000 --- a/src/SIL.Machine.AspNetCore/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,2 +0,0 @@ -[assembly: InternalsVisibleTo("SIL.Machine.AspNetCore.Tests")] -[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] diff --git a/src/SIL.Machine.AspNetCore/SIL.Machine.AspNetCore.csproj b/src/SIL.Machine.AspNetCore/SIL.Machine.AspNetCore.csproj deleted file mode 100644 index 159e7bab9..000000000 --- a/src/SIL.Machine.AspNetCore/SIL.Machine.AspNetCore.csproj +++ /dev/null @@ -1,61 +0,0 @@ - - - - net8.0 - An ASP.NET Core web API middleware for the Machine library. - enable - enable - true - true - true - $(NoWarn);CS1591;CS1573 - - - - - - - - - - - PreserveNewest - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/SIL.Machine.AspNetCore/Services/BuildJobService.cs b/src/SIL.Machine.AspNetCore/Services/BuildJobService.cs deleted file mode 100644 index 406474283..000000000 --- a/src/SIL.Machine.AspNetCore/Services/BuildJobService.cs +++ /dev/null @@ -1,211 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class BuildJobService(IEnumerable runners, IRepository engines) - : IBuildJobService -{ - private readonly Dictionary _runners = runners.ToDictionary(r => r.Type); - private readonly IRepository _engines = engines; - - public Task IsEngineBuilding(string engineId, CancellationToken cancellationToken = default) - { - return _engines.ExistsAsync(e => e.EngineId == engineId && e.CurrentBuild != null, cancellationToken); - } - - public Task> GetBuildingEnginesAsync( - BuildJobRunnerType runner, - CancellationToken cancellationToken = default - ) - { - return _engines.GetAllAsync( - e => e.CurrentBuild != null && e.CurrentBuild.BuildJobRunner == runner, - cancellationToken - ); - } - - public async Task GetBuildAsync( - string engineId, - string buildId, - CancellationToken cancellationToken = default - ) - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.CurrentBuild != null && e.CurrentBuild.BuildId == buildId, - cancellationToken - ); - return engine?.CurrentBuild; - } - - public async Task CreateEngineAsync( - string engineId, - string? name = null, - CancellationToken cancellationToken = default - ) - { - foreach (BuildJobRunnerType runnerType in _runners.Keys) - { - IBuildJobRunner runner = _runners[runnerType]; - await runner.CreateEngineAsync(engineId, name, cancellationToken); - } - } - - public async Task DeleteEngineAsync(string engineId, CancellationToken cancellationToken = default) - { - foreach (BuildJobRunnerType runnerType in _runners.Keys) - { - IBuildJobRunner runner = _runners[runnerType]; - await runner.DeleteEngineAsync(engineId, cancellationToken); - } - } - - public async Task StartBuildJobAsync( - BuildJobRunnerType runnerType, - string engineId, - string buildId, - BuildStage stage, - object? data = null, - string? buildOptions = null, - CancellationToken cancellationToken = default - ) - { - TranslationEngine? engine = await _engines.GetAsync( - e => - e.EngineId == engineId - && (e.CurrentBuild == null || e.CurrentBuild.JobState != BuildJobState.Canceling), - cancellationToken - ); - if (engine is null) - return false; - - IBuildJobRunner runner = _runners[runnerType]; - string jobId = await runner.CreateJobAsync( - engine.Type, - engineId, - buildId, - stage, - data, - buildOptions, - cancellationToken - ); - try - { - await _engines.UpdateAsync( - e => e.EngineId == engineId, - u => - u.Set( - e => e.CurrentBuild, - new Build - { - BuildId = buildId, - JobId = jobId, - BuildJobRunner = runner.Type, - Stage = stage, - JobState = BuildJobState.Pending, - Options = buildOptions - } - ), - cancellationToken: cancellationToken - ); - await runner.EnqueueJobAsync(jobId, engine.Type, cancellationToken); - return true; - } - catch - { - await runner.DeleteJobAsync(jobId, CancellationToken.None); - throw; - } - } - - public async Task<(string? BuildId, BuildJobState State)> CancelBuildJobAsync( - string engineId, - CancellationToken cancellationToken = default - ) - { - TranslationEngine? engine = await _engines.GetAsync( - e => e.EngineId == engineId && e.CurrentBuild != null, - cancellationToken - ); - if (engine is null || engine.CurrentBuild is null) - return (null, BuildJobState.None); - - IBuildJobRunner runner = _runners[engine.CurrentBuild.BuildJobRunner]; - - if (engine.CurrentBuild.JobState is BuildJobState.Pending) - { - // cancel a job that hasn't started yet - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.CurrentBuild != null, - u => u.Unset(b => b.CurrentBuild), - returnOriginal: true, - cancellationToken: cancellationToken - ); - if (engine is not null && engine.CurrentBuild is not null) - { - // job will be deleted from the queue - await runner.StopJobAsync(engine.CurrentBuild.JobId, CancellationToken.None); - return (engine.CurrentBuild.BuildId, BuildJobState.None); - } - } - else if (engine.CurrentBuild.JobState is BuildJobState.Active) - { - // cancel a job that is already running - engine = await _engines.UpdateAsync( - e => e.EngineId == engineId && e.CurrentBuild != null, - u => u.Set(e => e.CurrentBuild!.JobState, BuildJobState.Canceling), - cancellationToken: cancellationToken - ); - if (engine is not null && engine.CurrentBuild is not null) - { - await runner.StopJobAsync(engine.CurrentBuild.JobId, CancellationToken.None); - return (engine.CurrentBuild.BuildId, BuildJobState.Canceling); - } - } - - return (null, BuildJobState.None); - } - - public async Task BuildJobStartedAsync( - string engineId, - string buildId, - CancellationToken cancellationToken = default - ) - { - TranslationEngine? engine = await _engines.UpdateAsync( - e => - e.EngineId == engineId - && e.CurrentBuild != null - && e.CurrentBuild.BuildId == buildId - && e.CurrentBuild.JobState == BuildJobState.Pending, - u => u.Set(e => e.CurrentBuild!.JobState, BuildJobState.Active), - cancellationToken: cancellationToken - ); - return engine is not null; - } - - public Task BuildJobFinishedAsync( - string engineId, - string buildId, - bool buildComplete, - CancellationToken cancellationToken = default - ) - { - return _engines.UpdateAsync( - e => e.EngineId == engineId && e.CurrentBuild != null && e.CurrentBuild.BuildId == buildId, - u => - { - u.Unset(e => e.CurrentBuild); - if (buildComplete) - u.Inc(e => e.BuildRevision); - }, - cancellationToken: cancellationToken - ); - } - - public Task BuildJobRestartingAsync(string engineId, string buildId, CancellationToken cancellationToken = default) - { - return _engines.UpdateAsync( - e => e.EngineId == engineId && e.CurrentBuild != null && e.CurrentBuild.BuildId == buildId, - u => u.Set(e => e.CurrentBuild!.JobState, BuildJobState.Pending), - cancellationToken: cancellationToken - ); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/BuildProgress.cs b/src/SIL.Machine.AspNetCore/Services/BuildProgress.cs deleted file mode 100644 index 44de1c7de..000000000 --- a/src/SIL.Machine.AspNetCore/Services/BuildProgress.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class BuildProgress(IPlatformService platformService, string buildId) : IProgress -{ - private readonly IPlatformService _platformService = platformService; - private readonly string _buildId = buildId; - private ProgressStatus _prevStatus; - - private DateTime _lastReportTime = DateTime.Now; - - private const float ThrottleTimeSeconds = 1; - - public void Report(ProgressStatus value) - { - if (_prevStatus.Equals(value)) - return; - - if (DateTime.Now < _lastReportTime.AddSeconds(ThrottleTimeSeconds)) - return; - - _lastReportTime = DateTime.Now; - _platformService.UpdateBuildStatusAsync(_buildId, value); - _prevStatus = value; - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/CancellationInterceptor.cs b/src/SIL.Machine.AspNetCore/Services/CancellationInterceptor.cs deleted file mode 100644 index 81b765054..000000000 --- a/src/SIL.Machine.AspNetCore/Services/CancellationInterceptor.cs +++ /dev/null @@ -1,30 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class CancellationInterceptor(ILogger logger) : Interceptor -{ - private readonly ILogger _logger = logger; - - public override async Task UnaryServerHandler( - TRequest request, - ServerCallContext context, - UnaryServerMethod continuation - ) - { - try - { - return await continuation(request, context); - } - catch (Exception ex) - { - if (ex is OperationCanceledException) - { - _logger.LogInformation("An operation was canceled."); - throw new RpcException(new Status(StatusCode.Cancelled, "An operation was canceled.")); - } - else - { - throw; - } - } - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/ClearMLAuthenticationService.cs b/src/SIL.Machine.AspNetCore/Services/ClearMLAuthenticationService.cs deleted file mode 100644 index 68c72a081..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ClearMLAuthenticationService.cs +++ /dev/null @@ -1,73 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLAuthenticationService( - IServiceProvider services, - IHttpClientFactory httpClientFactory, - IOptionsMonitor options, - ILogger logger -) : RecurrentTask("ClearML authentication service", services, RefreshPeriod, logger), IClearMLAuthenticationService -{ - private readonly HttpClient _httpClient = httpClientFactory.CreateClient("ClearML"); - private readonly IOptionsMonitor _options = options; - private readonly ILogger _logger = logger; - private readonly AsyncLock _lock = new(); - - // technically, the token should be good for 30 days, but let's refresh each hour - // to know well ahead of time if something is wrong. - private static readonly TimeSpan RefreshPeriod = TimeSpan.FromSeconds(3600); - private string _authToken = ""; - - public async Task GetAuthTokenAsync(CancellationToken cancellationToken = default) - { - using (await _lock.LockAsync(cancellationToken)) - { - if (_authToken is "") - { - //Should only happen once, so no different in cost than previous solution - _logger.LogInformation("Token was empty; refreshing"); - await AuthorizeAsync(cancellationToken); - } - } - return _authToken; - } - - protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken) - { - try - { - using (await _lock.LockAsync(cancellationToken)) - await AuthorizeAsync(cancellationToken); - } - catch (Exception e) - { - if (_authToken is "") - { - _logger.LogError(e, "Error occurred while acquiring ClearML authentication token for the first time."); - // The ClearML token never was set. We can't continue without it. - throw; - } - else - { - _logger.LogError(e, "Error occurred while refreshing ClearML authentication token."); - } - } - } - - private async Task AuthorizeAsync(CancellationToken cancellationToken) - { - var request = new HttpRequestMessage(HttpMethod.Post, "auth.login") - { - Content = new StringContent("{}", Encoding.UTF8, "application/json") - }; - var authenticationString = $"{_options.CurrentValue.AccessKey}:{_options.CurrentValue.SecretKey}"; - var base64EncodedAuthenticationString = Convert.ToBase64String(Encoding.ASCII.GetBytes(authenticationString)); - request.Headers.Add("Authorization", $"Basic {base64EncodedAuthenticationString}"); - HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); - string result = await response.Content.ReadAsStringAsync(cancellationToken); - string? refreshedToken = (string?)((JsonObject?)JsonNode.Parse(result))?["data"]?["token"]; - if (refreshedToken is null || refreshedToken is "") - throw new Exception($"ClearML authentication failed - {response.StatusCode}: {response.ReasonPhrase}"); - _authToken = refreshedToken; - _logger.LogInformation("ClearML Authentication Token Refresh Successful."); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/ClearMLBuildJobRunner.cs b/src/SIL.Machine.AspNetCore/Services/ClearMLBuildJobRunner.cs deleted file mode 100644 index 234f05e3b..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ClearMLBuildJobRunner.cs +++ /dev/null @@ -1,88 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLBuildJobRunner( - IClearMLService clearMLService, - IEnumerable buildJobFactories, - IOptionsMonitor options -) : IBuildJobRunner -{ - private readonly IClearMLService _clearMLService = clearMLService; - private readonly Dictionary _buildJobFactories = - buildJobFactories.ToDictionary(f => f.EngineType); - - private readonly Dictionary _options = - options.CurrentValue.ClearML.ToDictionary(o => o.TranslationEngineType); - - public BuildJobRunnerType Type => BuildJobRunnerType.ClearML; - - public async Task CreateEngineAsync( - string engineId, - string? name = null, - CancellationToken cancellationToken = default - ) - { - await _clearMLService.CreateProjectAsync(engineId, name, cancellationToken); - } - - public async Task DeleteEngineAsync(string engineId, CancellationToken cancellationToken = default) - { - string? projectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - if (projectId is not null) - await _clearMLService.DeleteProjectAsync(projectId, cancellationToken); - } - - public async Task CreateJobAsync( - TranslationEngineType engineType, - string engineId, - string buildId, - BuildStage stage, - object? data = null, - string? buildOptions = null, - CancellationToken cancellationToken = default - ) - { - string? projectId = await _clearMLService.GetProjectIdAsync(engineId, cancellationToken); - projectId ??= await _clearMLService.CreateProjectAsync(engineId, cancellationToken: cancellationToken); - - ClearMLTask? task = await _clearMLService.GetTaskByNameAsync(buildId, cancellationToken); - if (task is not null) - return task.Id; - - IClearMLBuildJobFactory buildJobFactory = _buildJobFactories[engineType]; - string script = await buildJobFactory.CreateJobScriptAsync( - engineId, - buildId, - _options[engineType].ModelType, - stage, - data, - buildOptions, - cancellationToken - ); - return await _clearMLService.CreateTaskAsync( - buildId, - projectId, - script, - _options[engineType].DockerImage, - cancellationToken - ); - } - - public Task DeleteJobAsync(string jobId, CancellationToken cancellationToken = default) - { - return _clearMLService.DeleteTaskAsync(jobId, cancellationToken); - } - - public Task EnqueueJobAsync( - string jobId, - TranslationEngineType engineType, - CancellationToken cancellationToken = default - ) - { - return _clearMLService.EnqueueTaskAsync(jobId, _options[engineType].Queue, cancellationToken); - } - - public Task StopJobAsync(string jobId, CancellationToken cancellationToken = default) - { - return _clearMLService.StopTaskAsync(jobId, cancellationToken); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/ClearMLHealthCheck.cs b/src/SIL.Machine.AspNetCore/Services/ClearMLHealthCheck.cs deleted file mode 100644 index 0cce9b7d1..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ClearMLHealthCheck.cs +++ /dev/null @@ -1,100 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLHealthCheck( - IClearMLAuthenticationService clearMLAuthenticationService, - IHttpClientFactory httpClientFactory, - IOptionsMonitor buildJobOptions -) : IHealthCheck -{ - private readonly HttpClient _httpClient = httpClientFactory.CreateClient("ClearML-NoRetry"); - private readonly IClearMLAuthenticationService _clearMLAuthenticationService = clearMLAuthenticationService; - private readonly ISet _queuesMonitored = buildJobOptions - .CurrentValue.ClearML.Select(x => x.Queue) - .ToHashSet(); - - private int _numConsecutiveFailures = 0; - private readonly AsyncLock _lock = new AsyncLock(); - - public async Task CheckHealthAsync( - HealthCheckContext context, - CancellationToken cancellationToken = default - ) - { - try - { - if (!await PingAsync(cancellationToken)) - return HealthCheckResult.Unhealthy("ClearML is unresponsive"); - IReadOnlySet queuesWithoutWorkers = await QueuesWithoutWorkers(cancellationToken); - if (queuesWithoutWorkers.Count > 0) - { - return HealthCheckResult.Unhealthy( - $"No ClearML agents are available for configured queues: {string.Join(", ", queuesWithoutWorkers)}" - ); - } - - using (await _lock.LockAsync()) - _numConsecutiveFailures = 0; - return HealthCheckResult.Healthy("ClearML is available"); - } - catch (Exception e) - { - using (await _lock.LockAsync()) - { - _numConsecutiveFailures++; - return _numConsecutiveFailures > 3 - ? HealthCheckResult.Unhealthy(exception: e) - : HealthCheckResult.Degraded(exception: e); - } - } - } - - private async Task CallAsync( - string service, - string action, - JsonNode body, - CancellationToken cancellationToken = default - ) - { - var request = new HttpRequestMessage(HttpMethod.Post, $"{service}.{action}") - { - Content = new StringContent(body.ToJsonString(), Encoding.UTF8, "application/json") - }; - request.Headers.Add( - "Authorization", - $"Bearer {await _clearMLAuthenticationService.GetAuthTokenAsync(cancellationToken)}" - ); - HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); - string result = await response.Content.ReadAsStringAsync(cancellationToken); - return (JsonObject?)JsonNode.Parse(result); - } - - public async Task PingAsync(CancellationToken cancellationToken = default) - { - JsonObject? result = await CallAsync("debug", "ping", new JsonObject(), cancellationToken); - return result is not null; - } - - public async Task> QueuesWithoutWorkers(CancellationToken cancellationToken = default) - { - HashSet queuesWithoutWorkers = _queuesMonitored.ToHashSet(); - JsonObject? result = await CallAsync("workers", "get_all", new JsonObject(), cancellationToken); - JsonNode? workers_node = result?["data"]?["workers"]; - if (workers_node is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - JsonArray workers = (JsonArray)workers_node; - foreach (var worker in workers) - { - JsonNode? queues_node = worker?["queues"]; - if (queues_node is null) - continue; - JsonArray queues = (JsonArray)queues_node; - foreach (var currentQueue in queues) - { - string? currentQueueName = (string?)currentQueue?["name"]; - if (currentQueueName is not null) - queuesWithoutWorkers.Remove(currentQueueName); - } - } - return queuesWithoutWorkers; - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/ClearMLMonitorService.cs b/src/SIL.Machine.AspNetCore/Services/ClearMLMonitorService.cs deleted file mode 100644 index 87e9ed8e9..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ClearMLMonitorService.cs +++ /dev/null @@ -1,407 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLMonitorService( - IServiceProvider services, - IClearMLService clearMLService, - ISharedFileService sharedFileService, - IOptionsMonitor clearMLOptions, - IOptionsMonitor buildJobOptions, - ILogger logger -) - : RecurrentTask( - "ClearML monitor service", - services, - clearMLOptions.CurrentValue.BuildPollingTimeout, - logger, - clearMLOptions.CurrentValue.BuildPollingEnabled - ), - IClearMLQueueService -{ - private static readonly string SummaryMetric = CreateMD5("Summary"); - private static readonly string TrainCorpusSizeVariant = CreateMD5("train_corpus_size"); - private static readonly string ConfidenceVariant = CreateMD5("confidence"); - - private readonly IClearMLService _clearMLService = clearMLService; - private readonly ISharedFileService _sharedFileService = sharedFileService; - private readonly ILogger _logger = logger; - private readonly Dictionary _curBuildStatus = new(); - - private readonly IReadOnlyDictionary _queuePerEngineType = - buildJobOptions.CurrentValue.ClearML.ToDictionary(x => x.TranslationEngineType, x => x.Queue); - - private readonly IDictionary _queueSizePerEngineType = new ConcurrentDictionary< - TranslationEngineType, - int - >(buildJobOptions.CurrentValue.ClearML.ToDictionary(x => x.TranslationEngineType, x => 0)); - - public int GetQueueSize(TranslationEngineType engineType) - { - return _queueSizePerEngineType[engineType]; - } - - protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken) - { - try - { - var buildJobService = scope.ServiceProvider.GetRequiredService(); - IReadOnlyList trainingEngines = await buildJobService.GetBuildingEnginesAsync( - BuildJobRunnerType.ClearML, - cancellationToken - ); - if (trainingEngines.Count == 0) - return; - - Dictionary tasks = new(); - Dictionary queuePositions = new(); - - foreach (TranslationEngineType engineType in _queuePerEngineType.Keys) - { - Dictionary tasksPerEngineType = ( - await _clearMLService.GetTasksByIdAsync( - trainingEngines.Select(e => e.CurrentBuild!.JobId), - cancellationToken - ) - ) - .UnionBy( - await _clearMLService.GetTasksForQueueAsync(_queuePerEngineType[engineType], cancellationToken), - t => t.Id - ) - .ToDictionary(t => t.Id); - // add new keys to dictionary - foreach (KeyValuePair kvp in tasksPerEngineType) - tasks.TryAdd(kvp.Key, kvp.Value); - - Dictionary queuePositionsPerEngineType = tasksPerEngineType - .Values.Where(t => t.Status is ClearMLTaskStatus.Queued or ClearMLTaskStatus.Created) - .OrderBy(t => t.Created) - .Select((t, i) => (Position: i, Task: t)) - .ToDictionary(e => e.Task.Name, e => e.Position); - // add new keys to dictionary - foreach (KeyValuePair kvp in queuePositionsPerEngineType) - queuePositions.TryAdd(kvp.Key, kvp.Value); - - _queueSizePerEngineType[engineType] = queuePositionsPerEngineType.Count; - } - - var dataAccessContext = scope.ServiceProvider.GetRequiredService(); - var platformService = scope.ServiceProvider.GetRequiredService(); - var lockFactory = scope.ServiceProvider.GetRequiredService(); - foreach (TranslationEngine engine in trainingEngines) - { - if (engine.CurrentBuild is null || !tasks.TryGetValue(engine.CurrentBuild.JobId, out ClearMLTask? task)) - continue; - - if ( - engine.CurrentBuild.JobState is BuildJobState.Pending - && task.Status is ClearMLTaskStatus.Queued or ClearMLTaskStatus.Created - ) - { - await UpdateTrainJobStatus( - platformService, - engine.CurrentBuild.BuildId, - new ProgressStatus(step: 0, percentCompleted: 0.0), - //CurrentBuild.BuildId should always equal the corresponding task.Name - queuePositions[engine.CurrentBuild.BuildId] + 1, - cancellationToken - ); - } - - if (engine.CurrentBuild.Stage == BuildStage.Train) - { - if ( - engine.CurrentBuild.JobState is BuildJobState.Pending - && task.Status - is ClearMLTaskStatus.InProgress - or ClearMLTaskStatus.Stopped - or ClearMLTaskStatus.Failed - or ClearMLTaskStatus.Completed - ) - { - bool canceled = !await TrainJobStartedAsync( - dataAccessContext, - lockFactory, - buildJobService, - platformService, - engine.EngineId, - engine.CurrentBuild.BuildId, - cancellationToken - ); - if (canceled) - continue; - } - - switch (task.Status) - { - case ClearMLTaskStatus.InProgress: - { - double? percentCompleted = null; - if (task.Runtime.TryGetValue("progress", out string? progressStr)) - percentCompleted = int.Parse(progressStr, CultureInfo.InvariantCulture) / 100.0; - task.Runtime.TryGetValue("message", out string? message); - await UpdateTrainJobStatus( - platformService, - engine.CurrentBuild.BuildId, - new ProgressStatus(task.LastIteration ?? 0, percentCompleted, message), - queueDepth: 0, - cancellationToken - ); - break; - } - - case ClearMLTaskStatus.Completed: - { - task.Runtime.TryGetValue("message", out string? message); - await UpdateTrainJobStatus( - platformService, - engine.CurrentBuild.BuildId, - new ProgressStatus(task.LastIteration ?? 0, percentCompleted: 1.0, message), - queueDepth: 0, - cancellationToken - ); - bool canceling = !await TrainJobCompletedAsync( - lockFactory, - buildJobService, - engine.EngineId, - engine.CurrentBuild.BuildId, - (int)GetMetric(task, SummaryMetric, TrainCorpusSizeVariant), - GetMetric(task, SummaryMetric, ConfidenceVariant), - engine.CurrentBuild.Options, - cancellationToken - ); - if (canceling) - { - await TrainJobCanceledAsync( - dataAccessContext, - lockFactory, - buildJobService, - platformService, - engine.EngineId, - engine.CurrentBuild.BuildId, - cancellationToken - ); - } - break; - } - - case ClearMLTaskStatus.Stopped: - { - await TrainJobCanceledAsync( - dataAccessContext, - lockFactory, - buildJobService, - platformService, - engine.EngineId, - engine.CurrentBuild.BuildId, - cancellationToken - ); - break; - } - - case ClearMLTaskStatus.Failed: - { - await TrainJobFaultedAsync( - dataAccessContext, - lockFactory, - buildJobService, - platformService, - engine.EngineId, - engine.CurrentBuild.BuildId, - $"{task.StatusReason} : {task.StatusMessage}", - cancellationToken - ); - break; - } - } - } - } - } - catch (Exception e) - { - _logger.LogError(e, "Error occurred while monitoring ClearML tasks."); - } - } - - private async Task TrainJobStartedAsync( - IDataAccessContext dataAccessContext, - IDistributedReaderWriterLockFactory lockFactory, - IBuildJobService buildJobService, - IPlatformService platformService, - string engineId, - string buildId, - CancellationToken cancellationToken = default - ) - { - bool success; - IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - success = await dataAccessContext.WithTransactionAsync( - async (ct) => - { - if (!await buildJobService.BuildJobStartedAsync(engineId, buildId, ct)) - return false; - await platformService.BuildStartedAsync(buildId, CancellationToken.None); - return true; - }, - cancellationToken: cancellationToken - ); - } - await UpdateTrainJobStatus(platformService, buildId, new ProgressStatus(0), 0, cancellationToken); - _logger.LogInformation("Build started ({BuildId})", buildId); - return success; - } - - private async Task TrainJobCompletedAsync( - IDistributedReaderWriterLockFactory lockFactory, - IBuildJobService buildJobService, - string engineId, - string buildId, - int corpusSize, - double confidence, - string? buildOptions, - CancellationToken cancellationToken - ) - { - try - { - IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - return await buildJobService.StartBuildJobAsync( - BuildJobRunnerType.Hangfire, - engineId, - buildId, - BuildStage.Postprocess, - (corpusSize, confidence), - buildOptions, - cancellationToken - ); - } - } - finally - { - _curBuildStatus.Remove(buildId); - } - } - - private async Task TrainJobFaultedAsync( - IDataAccessContext dataAccessContext, - IDistributedReaderWriterLockFactory lockFactory, - IBuildJobService buildJobService, - IPlatformService platformService, - string engineId, - string buildId, - string message, - CancellationToken cancellationToken - ) - { - try - { - IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await dataAccessContext.WithTransactionAsync( - async (ct) => - { - await platformService.BuildFaultedAsync(buildId, message, ct); - await buildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: false, - CancellationToken.None - ); - }, - cancellationToken: cancellationToken - ); - } - _logger.LogError("Build faulted ({BuildId}). Error: {ErrorMessage}", buildId, message); - } - finally - { - _curBuildStatus.Remove(buildId); - } - } - - private async Task TrainJobCanceledAsync( - IDataAccessContext dataAccessContext, - IDistributedReaderWriterLockFactory lockFactory, - IBuildJobService buildJobService, - IPlatformService platformService, - string engineId, - string buildId, - CancellationToken cancellationToken - ) - { - try - { - IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await dataAccessContext.WithTransactionAsync( - async (ct) => - { - await platformService.BuildCanceledAsync(buildId, ct); - await buildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: false, - CancellationToken.None - ); - }, - cancellationToken: cancellationToken - ); - } - _logger.LogInformation("Build canceled ({BuildId})", buildId); - } - finally - { - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/", CancellationToken.None); - } - catch (Exception e) - { - _logger.LogWarning(e, "Unable to to delete job data for build {BuildId}.", buildId); - } - _curBuildStatus.Remove(buildId); - } - } - - private async Task UpdateTrainJobStatus( - IPlatformService platformService, - string buildId, - ProgressStatus progressStatus, - int? queueDepth = null, - CancellationToken cancellationToken = default - ) - { - if ( - _curBuildStatus.TryGetValue(buildId, out ProgressStatus curProgressStatus) - && curProgressStatus.Equals(progressStatus) - ) - { - return; - } - await platformService.UpdateBuildStatusAsync(buildId, progressStatus, queueDepth, cancellationToken); - _curBuildStatus[buildId] = progressStatus; - } - - private static double GetMetric(ClearMLTask task, string metric, string variant) - { - if (!task.LastMetrics.TryGetValue(metric, out IReadOnlyDictionary? metricVariants)) - return 0; - - if (!metricVariants.TryGetValue(variant, out ClearMLMetricsEvent? metricEvent)) - return 0; - - return metricEvent.Value; - } - - private static string CreateMD5(string input) - { - byte[] inputBytes = Encoding.UTF8.GetBytes(input); - byte[] hashBytes = MD5.HashData(inputBytes); - - return Convert.ToHexString(hashBytes).ToLower(); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/ClearMLService.cs b/src/SIL.Machine.AspNetCore/Services/ClearMLService.cs deleted file mode 100644 index fd11b25dd..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ClearMLService.cs +++ /dev/null @@ -1,221 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ClearMLService( - IHttpClientFactory httpClientFactory, - IOptionsMonitor options, - IClearMLAuthenticationService clearMLAuthService, - IHostEnvironment env -) : IClearMLService -{ - private readonly HttpClient _httpClient = httpClientFactory.CreateClient("ClearML"); - private readonly IOptionsMonitor _options = options; - private readonly IHostEnvironment _env = env; - private static readonly JsonNamingPolicy JsonNamingPolicy = new SnakeCaseJsonNamingPolicy(); - private static readonly JsonSerializerOptions JsonSerializerOptions = - new() - { - PropertyNamingPolicy = JsonNamingPolicy, - Converters = { new CustomEnumConverterFactory(JsonNamingPolicy) } - }; - - private readonly IClearMLAuthenticationService _clearMLAuthService = clearMLAuthService; - - public async Task GetProjectIdAsync(string name, CancellationToken cancellationToken = default) - { - var body = new JsonObject - { - ["name"] = $"{_options.CurrentValue.RootProject}/{_options.CurrentValue.Project}/{name}", - ["only_fields"] = new JsonArray("id") - }; - JsonObject? result = await CallAsync("projects", "get_all", body, cancellationToken); - var projects = (JsonArray?)result?["data"]?["projects"]; - if (projects is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - if (projects.Count == 0) - return null; - return (string?)projects[0]?["id"]; - } - - public async Task CreateProjectAsync( - string name, - string? description = null, - CancellationToken cancellationToken = default - ) - { - var body = new JsonObject - { - ["name"] = $"{_options.CurrentValue.RootProject}/{_options.CurrentValue.Project}/{name}" - }; - if (description != null) - body["description"] = description; - JsonObject? result = await CallAsync("projects", "create", body, cancellationToken); - var projectId = (string?)result?["data"]?["id"]; - if (projectId is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - return projectId; - } - - public async Task DeleteProjectAsync(string id, CancellationToken cancellationToken = default) - { - var body = new JsonObject - { - ["project"] = id, - ["delete_contents"] = true, - ["force"] = true // needed if there are tasks already in that project. - }; - JsonObject? result = await CallAsync("projects", "delete", body, cancellationToken); - var deleted = (int?)result?["data"]?["deleted"]; - if (deleted is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - return deleted == 1; - } - - public async Task CreateTaskAsync( - string buildId, - string projectId, - string script, - string dockerImage, - CancellationToken cancellationToken = default - ) - { - var snakeCaseEnvironment = JsonNamingPolicy.ConvertName(_env.EnvironmentName); - var body = new JsonObject - { - ["name"] = buildId, - ["project"] = projectId, - ["script"] = new JsonObject { ["diff"] = script }, - ["container"] = new JsonObject - { - ["image"] = dockerImage, - ["arguments"] = "--env ENV_FOR_DYNACONF=" + snakeCaseEnvironment, - }, - ["type"] = "training" - }; - JsonObject? result = await CallAsync("tasks", "create", body, cancellationToken); - var taskId = (string?)result?["data"]?["id"]; - if (taskId is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - return taskId; - } - - public async Task DeleteTaskAsync(string id, CancellationToken cancellationToken = default) - { - var body = new JsonObject { ["task"] = id }; - JsonObject? result = await CallAsync("tasks", "delete", body, cancellationToken); - var deleted = (bool?)result?["data"]?["deleted"]; - if (deleted is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - return deleted.Value; - } - - public async Task EnqueueTaskAsync(string id, string queue, CancellationToken cancellationToken = default) - { - var body = new JsonObject { ["task"] = id, ["queue_name"] = queue }; - JsonObject? result = await CallAsync("tasks", "enqueue", body, cancellationToken); - var queued = (int?)result?["data"]?["queued"]; - if (queued is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - return queued == 1; - } - - public async Task DequeueTaskAsync(string id, CancellationToken cancellationToken = default) - { - var body = new JsonObject { ["task"] = id }; - JsonObject? result = await CallAsync("tasks", "dequeue", body, cancellationToken); - var dequeued = (int?)result?["data"]?["dequeued"]; - if (dequeued is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - return dequeued == 1; - } - - public async Task StopTaskAsync(string id, CancellationToken cancellationToken = default) - { - var body = new JsonObject { ["task"] = id, ["force"] = true }; - JsonObject? result = await CallAsync("tasks", "stop", body, cancellationToken); - var updated = (int?)result?["data"]?["updated"]; - if (updated is null) - throw new InvalidOperationException("Malformed response from ClearML server."); - return updated == 1; - } - - public async Task> GetTasksForQueueAsync( - string queue, - CancellationToken cancellationToken = default - ) - { - var body = new JsonObject { ["name"] = queue }; - JsonObject? result = await CallAsync("queues", "get_all_ex", body, cancellationToken); - var tasks = (JsonArray?)result?["data"]?["queues"]?[0]?["entries"]; - IEnumerable taskIds = tasks?.Select(t => (string)t?["id"]!) ?? new List(); - return await GetTasksByIdAsync(taskIds, cancellationToken); - } - - public async Task GetTaskByNameAsync(string name, CancellationToken cancellationToken = default) - { - IReadOnlyList tasks = await GetTasksAsync(new JsonObject { ["name"] = name }, cancellationToken); - if (tasks.Count == 0) - return null; - return tasks[0]; - } - - public Task> GetTasksByIdAsync( - IEnumerable ids, - CancellationToken cancellationToken = default - ) - { - return GetTasksAsync(new JsonObject { ["id"] = JsonValue.Create(ids.ToArray()) }, cancellationToken); - } - - private async Task> GetTasksAsync( - JsonObject body, - CancellationToken cancellationToken = default - ) - { - body["only_fields"] = new JsonArray( - "id", - "name", - "status", - "project", - "last_iteration", - "status_reason", - "status_message", - "created", - "active_duration", - "last_metrics", - "runtime" - ); - JsonObject? result = await CallAsync("tasks", "get_all_ex", body, cancellationToken); - var tasks = (JsonArray?)result?["data"]?["tasks"]; - return tasks?.Select(t => t.Deserialize(JsonSerializerOptions)!).ToArray() - ?? Array.Empty(); - } - - private async Task CallAsync( - string service, - string action, - JsonNode body, - CancellationToken cancellationToken = default - ) - { - var request = new HttpRequestMessage(HttpMethod.Post, $"{service}.{action}") - { - Content = new StringContent(body.ToJsonString(), Encoding.UTF8, "application/json") - }; - request.Headers.Add( - "Authorization", - $"Bearer {await _clearMLAuthService.GetAuthTokenAsync(cancellationToken)}" - ); - HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); - string result = await response.Content.ReadAsStringAsync(cancellationToken); - return (JsonObject?)JsonNode.Parse(result); - } - - private class SnakeCaseJsonNamingPolicy : JsonNamingPolicy - { - public override string ConvertName(string name) - { - return string.Concat(name.Select((x, i) => i > 0 && char.IsUpper(x) ? "_" + x.ToString() : x.ToString())) - .ToLowerInvariant(); - } - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/CorpusService.cs b/src/SIL.Machine.AspNetCore/Services/CorpusService.cs deleted file mode 100644 index 635bbff5a..000000000 --- a/src/SIL.Machine.AspNetCore/Services/CorpusService.cs +++ /dev/null @@ -1,51 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class CorpusService : ICorpusService -{ - public IEnumerable CreateTextCorpora(IReadOnlyList files) - { - List corpora = []; - - List> textFileCorpora = []; - foreach (CorpusFile file in files) - { - switch (file.Format) - { - case FileFormat.Text: - // if there are multiple texts with the same id, then add it to a new corpus or the first - // corpus that doesn't contain a text with that id - Dictionary? corpus = textFileCorpora.FirstOrDefault(c => - !c.ContainsKey(file.TextId) - ); - if (corpus is null) - { - corpus = []; - textFileCorpora.Add(corpus); - } - corpus[file.TextId] = new TextFileText(file.TextId, file.Location); - break; - - case FileFormat.Paratext: - corpora.Add(new ParatextBackupTextCorpus(file.Location, includeAllText: true)); - break; - } - } - foreach (Dictionary corpus in textFileCorpora) - corpora.Add(new DictionaryTextCorpus(corpus.Values)); - - return corpora; - } - - public IEnumerable CreateTermCorpora(IReadOnlyList files) - { - foreach (CorpusFile file in files) - { - switch (file.Format) - { - case FileFormat.Paratext: - yield return new ParatextBackupTermsCorpus(file.Location, ["PN"]); - break; - } - } - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/DistributedReaderWriterLock.cs b/src/SIL.Machine.AspNetCore/Services/DistributedReaderWriterLock.cs deleted file mode 100644 index 2653c772a..000000000 --- a/src/SIL.Machine.AspNetCore/Services/DistributedReaderWriterLock.cs +++ /dev/null @@ -1,177 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class DistributedReaderWriterLock(string hostId, IRepository locks, IIdGenerator idGenerator, string id) - : IDistributedReaderWriterLock -{ - private readonly string _hostId = hostId; - private readonly IRepository _locks = locks; - private readonly IIdGenerator _idGenerator = idGenerator; - private readonly string _id = id; - - public async Task ReaderLockAsync( - TimeSpan? lifetime = default, - CancellationToken cancellationToken = default - ) - { - string lockId = _idGenerator.GenerateId(); - if (!await TryAcquireReaderLock(lockId, lifetime, cancellationToken)) - { - using ISubscription sub = await _locks.SubscribeAsync(rwl => rwl.Id == _id, cancellationToken); - do - { - RWLock? rwLock = sub.Change.Entity; - if (rwLock is not null && !rwLock.IsAvailableForReading()) - { - TimeSpan? timeout = default; - if (rwLock.WriterLock?.ExpiresAt is not null) - { - timeout = rwLock.WriterLock.ExpiresAt - DateTime.UtcNow; - if (timeout < TimeSpan.Zero) - timeout = TimeSpan.Zero; - } - if (timeout != TimeSpan.Zero) - await sub.WaitForChangeAsync(timeout, cancellationToken); - } - } while (!await TryAcquireReaderLock(lockId, lifetime, cancellationToken)); - } - return new ReaderLockReleaser(this, lockId); - } - - public async Task WriterLockAsync( - TimeSpan? lifetime = default, - CancellationToken cancellationToken = default - ) - { - string lockId = _idGenerator.GenerateId(); - if (!await TryAcquireWriterLock(lockId, lifetime, cancellationToken)) - { - await _locks.UpdateAsync( - _id, - u => u.Add(rwl => rwl.WriterQueue, new Lock { Id = lockId, HostId = _hostId }), - cancellationToken: cancellationToken - ); - try - { - using ISubscription sub = await _locks.SubscribeAsync(rwl => rwl.Id == _id, cancellationToken); - do - { - RWLock? rwLock = sub.Change.Entity; - if (rwLock is not null && !rwLock.IsAvailableForWriting(lockId)) - { - List dateTimes = rwLock - .ReaderLocks.Where(l => l.ExpiresAt.HasValue) - .Select(l => l.ExpiresAt.GetValueOrDefault()) - .ToList(); - if (rwLock.WriterLock?.ExpiresAt is not null) - dateTimes.Add(rwLock.WriterLock.ExpiresAt.Value); - TimeSpan? timeout = default; - if (dateTimes.Count > 0) - { - timeout = dateTimes.Max() - DateTime.UtcNow; - if (timeout < TimeSpan.Zero) - timeout = TimeSpan.Zero; - } - if (timeout != TimeSpan.Zero) - await sub.WaitForChangeAsync(timeout, cancellationToken); - } - } while (!await TryAcquireWriterLock(lockId, lifetime, cancellationToken)); - } - catch - { - await _locks.UpdateAsync( - _id, - u => u.RemoveAll(rwl => rwl.WriterQueue, l => l.Id == lockId), - cancellationToken: cancellationToken - ); - throw; - } - } - return new WriterLockReleaser(this, lockId); - } - - private async Task TryAcquireWriterLock( - string lockId, - TimeSpan? lifetime, - CancellationToken cancellationToken - ) - { - var now = DateTime.UtcNow; - Expression> filter = rwl => - rwl.Id == _id - && (rwl.WriterLock == null || rwl.WriterLock.ExpiresAt != null && rwl.WriterLock.ExpiresAt <= now) - && !rwl.ReaderLocks.Any(l => l.ExpiresAt == null || l.ExpiresAt > now) - && (!rwl.WriterQueue.Any() || rwl.WriterQueue[0].Id == lockId); - void Update(IUpdateBuilder u) - { - u.Set( - rwl => rwl.WriterLock, - new Lock - { - Id = lockId, - ExpiresAt = lifetime is null ? null : now + lifetime, - HostId = _hostId - } - ); - u.RemoveAll(rwl => rwl.WriterQueue, l => l.Id == lockId); - } - RWLock? rwLock = await _locks.UpdateAsync(filter, Update, cancellationToken: cancellationToken); - return rwLock is not null; - } - - private async Task TryAcquireReaderLock( - string lockId, - TimeSpan? lifetime, - CancellationToken cancellationToken - ) - { - var now = DateTime.UtcNow; - Expression> filter = rwl => - rwl.Id == _id - && (rwl.WriterLock == null || rwl.WriterLock.ExpiresAt != null && rwl.WriterLock.ExpiresAt <= now) - && !rwl.WriterQueue.Any(); - void Update(IUpdateBuilder u) - { - u.Add( - rwl => rwl.ReaderLocks, - new Lock - { - Id = lockId, - ExpiresAt = lifetime is null ? null : now + lifetime, - HostId = _hostId - } - ); - } - - RWLock? rwLock = await _locks.UpdateAsync(filter, Update, cancellationToken: cancellationToken); - return rwLock is not null; - } - - private class WriterLockReleaser(DistributedReaderWriterLock distributedLock, string lockId) : AsyncDisposableBase - { - private readonly DistributedReaderWriterLock _distributedLock = distributedLock; - private readonly string _lockId = lockId; - - protected override async ValueTask DisposeAsyncCore() - { - Expression> filter = rwl => - rwl.Id == _distributedLock._id && rwl.WriterLock != null && rwl.WriterLock.Id == _lockId; - await _distributedLock._locks.UpdateAsync(filter, u => u.Unset(rwl => rwl.WriterLock)); - } - } - - private class ReaderLockReleaser(DistributedReaderWriterLock distributedLock, string lockId) : AsyncDisposableBase - { - private readonly DistributedReaderWriterLock _distributedLock = distributedLock; - private readonly string _lockId = lockId; - - protected override async ValueTask DisposeAsyncCore() - { - Expression> filter = rwl => - rwl.Id == _distributedLock._id && rwl.ReaderLocks.Any(l => l.Id == _lockId); - await _distributedLock._locks.UpdateAsync( - filter, - u => u.RemoveAll(rwl => rwl.ReaderLocks, l => l.Id == _lockId) - ); - } - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/DistributedReaderWriterLockFactory.cs b/src/SIL.Machine.AspNetCore/Services/DistributedReaderWriterLockFactory.cs deleted file mode 100644 index 64a3c3aa5..000000000 --- a/src/SIL.Machine.AspNetCore/Services/DistributedReaderWriterLockFactory.cs +++ /dev/null @@ -1,77 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class DistributedReaderWriterLockFactory( - IOptions serviceOptions, - IRepository locks, - IIdGenerator idGenerator -) : IDistributedReaderWriterLockFactory -{ - private readonly ServiceOptions _serviceOptions = serviceOptions.Value; - private readonly IIdGenerator _idGenerator = idGenerator; - private readonly IRepository _locks = locks; - - public async Task InitAsync(CancellationToken cancellationToken = default) - { - await RemoveAllWaitersAsync(cancellationToken); - await ReleaseAllWriterLocksAsync(cancellationToken); - await ReleaseAllReaderLocksAsync(cancellationToken); - } - - public async Task CreateAsync( - string id, - CancellationToken cancellationToken = default - ) - { - try - { - await _locks.InsertAsync( - new RWLock - { - Id = id, - ReaderLocks = [], - WriterQueue = [] - }, - cancellationToken - ); - } - catch (DuplicateKeyException) - { - // the lock is already made - no new one needs to be made - // This is done instead of checking if it exists first to prevent race conditions. - } - return new DistributedReaderWriterLock(_serviceOptions.ServiceId, _locks, _idGenerator, id); - } - - public async Task DeleteAsync(string id, CancellationToken cancellationToken = default) - { - RWLock? rwLock = await _locks.DeleteAsync(rwl => rwl.Id == id, cancellationToken); - return rwLock is not null; - } - - private async Task ReleaseAllWriterLocksAsync(CancellationToken cancellationToken) - { - await _locks.UpdateAllAsync( - rwl => rwl.WriterLock != null && rwl.WriterLock.HostId == _serviceOptions.ServiceId, - u => u.Unset(rwl => rwl.WriterLock), - cancellationToken - ); - } - - private async Task ReleaseAllReaderLocksAsync(CancellationToken cancellationToken) - { - await _locks.UpdateAllAsync( - rwl => rwl.ReaderLocks.Any(l => l.HostId == _serviceOptions.ServiceId), - u => u.RemoveAll(rwl => rwl.ReaderLocks, l => l.HostId == _serviceOptions.ServiceId), - cancellationToken - ); - } - - private async Task RemoveAllWaitersAsync(CancellationToken cancellationToken) - { - await _locks.UpdateAllAsync( - rwl => rwl.WriterQueue.Any(l => l.HostId == _serviceOptions.ServiceId), - u => u.RemoveAll(rwl => rwl.WriterQueue, l => l.HostId == _serviceOptions.ServiceId), - cancellationToken - ); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/FileSystem.cs b/src/SIL.Machine.AspNetCore/Services/FileSystem.cs deleted file mode 100644 index a6b5367c1..000000000 --- a/src/SIL.Machine.AspNetCore/Services/FileSystem.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class FileSystem : IFileSystem -{ - public void CreateDirectory(string path) - { - Directory.CreateDirectory(path); - } - - public void DeleteFile(string path) - { - if (File.Exists(path)) - File.Delete(path); - } - - public Stream OpenWrite(string path) - { - return File.OpenWrite(path); - } - - public Stream OpenRead(string path) - { - return File.OpenRead(path); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/HangfireBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/HangfireBuildJob.cs deleted file mode 100644 index 515f88732..000000000 --- a/src/SIL.Machine.AspNetCore/Services/HangfireBuildJob.cs +++ /dev/null @@ -1,180 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public abstract class HangfireBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, - ILogger logger -) : HangfireBuildJob(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) -{ - public virtual Task RunAsync( - string engineId, - string buildId, - string? buildOptions, - CancellationToken cancellationToken - ) - { - return RunAsync(engineId, buildId, null, buildOptions, cancellationToken); - } -} - -public abstract class HangfireBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, - ILogger> logger -) -{ - protected IPlatformService PlatformService { get; } = platformService; - protected IRepository Engines { get; } = engines; - protected IDistributedReaderWriterLockFactory LockFactory { get; } = lockFactory; - protected IDataAccessContext DataAccessContext { get; } = dataAccessContext; - protected IBuildJobService BuildJobService { get; } = buildJobService; - protected ILogger> Logger { get; } = logger; - - public virtual async Task RunAsync( - string engineId, - string buildId, - T data, - string? buildOptions, - CancellationToken cancellationToken - ) - { - IDistributedReaderWriterLock @lock = await LockFactory.CreateAsync(engineId, cancellationToken); - JobCompletionStatus completionStatus = JobCompletionStatus.Completed; - try - { - await InitializeAsync(engineId, buildId, data, @lock, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - if (!await BuildJobService.BuildJobStartedAsync(engineId, buildId, cancellationToken)) - { - completionStatus = JobCompletionStatus.Canceled; - return; - } - } - - await DoWorkAsync(engineId, buildId, data, buildOptions, @lock, cancellationToken); - } - catch (OperationCanceledException) - { - // Check if the cancellation was initiated by an API call or a shutdown. - TranslationEngine? engine = await Engines.GetAsync( - e => e.EngineId == engineId && e.CurrentBuild != null && e.CurrentBuild.BuildId == buildId, - CancellationToken.None - ); - if (engine?.CurrentBuild?.JobState is BuildJobState.Canceling) - { - completionStatus = JobCompletionStatus.Canceled; - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - await DataAccessContext.WithTransactionAsync( - async (ct) => - { - await PlatformService.BuildCanceledAsync(buildId, CancellationToken.None); - await BuildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: false, - CancellationToken.None - ); - }, - cancellationToken: CancellationToken.None - ); - } - Logger.LogInformation("Build canceled ({0})", buildId); - } - else if (engine is not null) - { - // the build was canceled, because of a server shutdown - // switch state back to pending - completionStatus = JobCompletionStatus.Restarting; - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - await DataAccessContext.WithTransactionAsync( - async (ct) => - { - await PlatformService.BuildRestartingAsync(buildId, CancellationToken.None); - await BuildJobService.BuildJobRestartingAsync(engineId, buildId, CancellationToken.None); - }, - cancellationToken: CancellationToken.None - ); - } - throw; - } - else - { - completionStatus = JobCompletionStatus.Canceled; - } - } - catch (Exception e) - { - completionStatus = JobCompletionStatus.Faulted; - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - await DataAccessContext.WithTransactionAsync( - async (ct) => - { - await PlatformService.BuildFaultedAsync(buildId, e.Message, CancellationToken.None); - await BuildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: false, - CancellationToken.None - ); - }, - cancellationToken: CancellationToken.None - ); - } - Logger.LogError(0, e, "Build faulted ({0})", buildId); - throw; - } - finally - { - await CleanupAsync(engineId, buildId, data, @lock, completionStatus); - } - } - - protected virtual Task InitializeAsync( - string engineId, - string buildId, - T data, - IDistributedReaderWriterLock @lock, - CancellationToken cancellationToken - ) - { - return Task.CompletedTask; - } - - protected abstract Task DoWorkAsync( - string engineId, - string buildId, - T data, - string? buildOptions, - IDistributedReaderWriterLock @lock, - CancellationToken cancellationToken - ); - - protected virtual Task CleanupAsync( - string engineId, - string buildId, - T data, - IDistributedReaderWriterLock @lock, - JobCompletionStatus completionStatus - ) - { - return Task.CompletedTask; - } - - protected enum JobCompletionStatus - { - Completed, - Faulted, - Canceled, - Restarting - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/HangfireBuildJobRunner.cs b/src/SIL.Machine.AspNetCore/Services/HangfireBuildJobRunner.cs deleted file mode 100644 index 9b5c62ff3..000000000 --- a/src/SIL.Machine.AspNetCore/Services/HangfireBuildJobRunner.cs +++ /dev/null @@ -1,85 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class HangfireBuildJobRunner( - IBackgroundJobClient jobClient, - IEnumerable buildJobFactories -) : IBuildJobRunner -{ - public static Job CreateJob( - string engineId, - string buildId, - string queue, - object? data, - string? buildOptions - ) - where TJob : HangfireBuildJob - { - ArgumentNullException.ThrowIfNull(data); - // Token "None" is used here because hangfire injects the proper cancellation token - return Job.FromExpression( - j => j.RunAsync(engineId, buildId, (TData)data, buildOptions, CancellationToken.None), - queue - ); - } - - public static Job CreateJob(string engineId, string buildId, string queue, string? buildOptions) - where TJob : HangfireBuildJob - { - // Token "None" is used here because hangfire injects the proper cancellation token - return Job.FromExpression( - j => j.RunAsync(engineId, buildId, buildOptions, CancellationToken.None), - queue - ); - } - - private readonly IBackgroundJobClient _jobClient = jobClient; - private readonly Dictionary _buildJobFactories = - buildJobFactories.ToDictionary(f => f.EngineType); - - public BuildJobRunnerType Type => BuildJobRunnerType.Hangfire; - - public Task CreateEngineAsync(string engineId, string? name = null, CancellationToken cancellationToken = default) - { - return Task.CompletedTask; - } - - public Task DeleteEngineAsync(string engineId, CancellationToken cancellationToken = default) - { - return Task.CompletedTask; - } - - public Task CreateJobAsync( - TranslationEngineType engineType, - string engineId, - string buildId, - BuildStage stage, - object? data = null, - string? buildOptions = null, - CancellationToken cancellationToken = default - ) - { - IHangfireBuildJobFactory buildJobFactory = _buildJobFactories[engineType]; - Job job = buildJobFactory.CreateJob(engineId, buildId, stage, data, buildOptions); - return Task.FromResult(_jobClient.Create(job, new ScheduledState(TimeSpan.FromDays(10000)))); - } - - public Task DeleteJobAsync(string jobId, CancellationToken cancellationToken = default) - { - return Task.FromResult(_jobClient.Delete(jobId)); - } - - public Task EnqueueJobAsync( - string jobId, - TranslationEngineType engineType, - CancellationToken cancellationToken = default - ) - { - return Task.FromResult(_jobClient.Requeue(jobId)); - } - - public Task StopJobAsync(string jobId, CancellationToken cancellationToken = default) - { - // Trigger the cancellation token for the job - return Task.FromResult(_jobClient.Delete(jobId)); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/HangfireHealthCheck.cs b/src/SIL.Machine.AspNetCore/Services/HangfireHealthCheck.cs deleted file mode 100644 index 185256756..000000000 --- a/src/SIL.Machine.AspNetCore/Services/HangfireHealthCheck.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class HangfireHealthCheck(JobStorage jobStorage, IOptions options) : IHealthCheck -{ - private readonly JobStorage _jobStorage = jobStorage; - private readonly IOptions _options = options; - - public Task CheckHealthAsync( - HealthCheckContext context, - CancellationToken cancellationToken = default - ) - { - if ( - _jobStorage - .GetMonitoringApi() - .Servers() - .Any(s => (DateTime.UtcNow - s.Heartbeat) < _options.Value.ServerTimeout) - ) - { - return Task.FromResult(HealthCheckResult.Healthy()); - } - - return Task.FromResult(HealthCheckResult.Unhealthy("There are no Hangfire servers running.")); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/IBuildJobRunner.cs b/src/SIL.Machine.AspNetCore/Services/IBuildJobRunner.cs deleted file mode 100644 index 1167efe1e..000000000 --- a/src/SIL.Machine.AspNetCore/Services/IBuildJobRunner.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface IBuildJobRunner -{ - BuildJobRunnerType Type { get; } - - Task CreateEngineAsync(string engineId, string? name = null, CancellationToken cancellationToken = default); - Task DeleteEngineAsync(string engineId, CancellationToken cancellationToken = default); - - Task CreateJobAsync( - TranslationEngineType engineType, - string engineId, - string buildId, - BuildStage stage, - object? data = null, - string? buildOptions = null, - CancellationToken cancellationToken = default - ); - - Task DeleteJobAsync(string jobId, CancellationToken cancellationToken = default); - - Task EnqueueJobAsync( - string jobId, - TranslationEngineType engineType, - CancellationToken cancellationToken = default - ); - - Task StopJobAsync(string jobId, CancellationToken cancellationToken = default); -} diff --git a/src/SIL.Machine.AspNetCore/Services/IBuildJobService.cs b/src/SIL.Machine.AspNetCore/Services/IBuildJobService.cs deleted file mode 100644 index 37b28c55b..000000000 --- a/src/SIL.Machine.AspNetCore/Services/IBuildJobService.cs +++ /dev/null @@ -1,41 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface IBuildJobService -{ - Task> GetBuildingEnginesAsync( - BuildJobRunnerType runner, - CancellationToken cancellationToken = default - ); - - Task IsEngineBuilding(string engineId, CancellationToken cancellationToken = default); - - Task CreateEngineAsync(string engineId, string? name = null, CancellationToken cancellationToken = default); - - Task DeleteEngineAsync(string engineId, CancellationToken cancellationToken = default); - - Task StartBuildJobAsync( - BuildJobRunnerType jobType, - string engineId, - string buildId, - BuildStage stage, - object? data = default, - string? buildOptions = default, - CancellationToken cancellationToken = default - ); - - Task<(string? BuildId, BuildJobState State)> CancelBuildJobAsync( - string engineId, - CancellationToken cancellationToken = default - ); - - Task BuildJobStartedAsync(string engineId, string buildId, CancellationToken cancellationToken = default); - - Task BuildJobFinishedAsync( - string engineId, - string buildId, - bool buildComplete, - CancellationToken cancellationToken = default - ); - - Task BuildJobRestartingAsync(string engineId, string buildId, CancellationToken cancellationToken = default); -} diff --git a/src/SIL.Machine.AspNetCore/Services/IClearMLAuthenticationService.cs b/src/SIL.Machine.AspNetCore/Services/IClearMLAuthenticationService.cs deleted file mode 100644 index 42960d705..000000000 --- a/src/SIL.Machine.AspNetCore/Services/IClearMLAuthenticationService.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface IClearMLAuthenticationService : IHostedService -{ - public Task GetAuthTokenAsync(CancellationToken cancellationToken = default); -} diff --git a/src/SIL.Machine.AspNetCore/Services/IClearMLBuildJobFactory.cs b/src/SIL.Machine.AspNetCore/Services/IClearMLBuildJobFactory.cs deleted file mode 100644 index eeb84149a..000000000 --- a/src/SIL.Machine.AspNetCore/Services/IClearMLBuildJobFactory.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface IClearMLBuildJobFactory -{ - TranslationEngineType EngineType { get; } - - Task CreateJobScriptAsync( - string engineId, - string buildId, - string modelType, - BuildStage stage, - object? data = null, - string? buildOptions = null, - CancellationToken cancellationToken = default - ); -} diff --git a/src/SIL.Machine.AspNetCore/Services/IClearMLQueueService.cs b/src/SIL.Machine.AspNetCore/Services/IClearMLQueueService.cs deleted file mode 100644 index 32bcf270a..000000000 --- a/src/SIL.Machine.AspNetCore/Services/IClearMLQueueService.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface IClearMLQueueService -{ - public int GetQueueSize(TranslationEngineType engineType); -} diff --git a/src/SIL.Machine.AspNetCore/Services/IClearMLService.cs b/src/SIL.Machine.AspNetCore/Services/IClearMLService.cs deleted file mode 100644 index bad8bf576..000000000 --- a/src/SIL.Machine.AspNetCore/Services/IClearMLService.cs +++ /dev/null @@ -1,30 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface IClearMLService -{ - Task CreateProjectAsync( - string name, - string? description = null, - CancellationToken cancellationToken = default - ); - Task DeleteProjectAsync(string id, CancellationToken cancellationToken = default); - Task GetProjectIdAsync(string name, CancellationToken cancellationToken = default); - - Task CreateTaskAsync( - string buildId, - string projectId, - string script, - string dockerImage, - CancellationToken cancellationToken = default - ); - Task DeleteTaskAsync(string id, CancellationToken cancellationToken = default); - Task EnqueueTaskAsync(string id, string queue, CancellationToken cancellationToken = default); - Task DequeueTaskAsync(string id, CancellationToken cancellationToken = default); - Task StopTaskAsync(string id, CancellationToken cancellationToken = default); - Task> GetTasksForQueueAsync(string queue, CancellationToken cancellationToken = default); - Task GetTaskByNameAsync(string name, CancellationToken cancellationToken = default); - Task> GetTasksByIdAsync( - IEnumerable ids, - CancellationToken cancellationToken = default - ); -} diff --git a/src/SIL.Machine.AspNetCore/Services/ICorpusService.cs b/src/SIL.Machine.AspNetCore/Services/ICorpusService.cs deleted file mode 100644 index 7a7211cbe..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ICorpusService.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface ICorpusService -{ - IEnumerable CreateTextCorpora(IReadOnlyList files); - IEnumerable CreateTermCorpora(IReadOnlyList files); -} diff --git a/src/SIL.Machine.AspNetCore/Services/IDistributedReaderWriterLock.cs b/src/SIL.Machine.AspNetCore/Services/IDistributedReaderWriterLock.cs deleted file mode 100644 index 50f981f8c..000000000 --- a/src/SIL.Machine.AspNetCore/Services/IDistributedReaderWriterLock.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface IDistributedReaderWriterLock -{ - Task ReaderLockAsync(TimeSpan? lifetime = default, CancellationToken cancellationToken = default); - Task WriterLockAsync(TimeSpan? lifetime = default, CancellationToken cancellationToken = default); -} diff --git a/src/SIL.Machine.AspNetCore/Services/IDistributedReaderWriterLockFactory.cs b/src/SIL.Machine.AspNetCore/Services/IDistributedReaderWriterLockFactory.cs deleted file mode 100644 index e133a7c10..000000000 --- a/src/SIL.Machine.AspNetCore/Services/IDistributedReaderWriterLockFactory.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface IDistributedReaderWriterLockFactory -{ - Task InitAsync(CancellationToken cancellationToken = default); - Task CreateAsync(string id, CancellationToken cancellationToken = default); - Task DeleteAsync(string id, CancellationToken cancellationToken = default); -} diff --git a/src/SIL.Machine.AspNetCore/Services/IFileStorage.cs b/src/SIL.Machine.AspNetCore/Services/IFileStorage.cs deleted file mode 100644 index 3417cffae..000000000 --- a/src/SIL.Machine.AspNetCore/Services/IFileStorage.cs +++ /dev/null @@ -1,20 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface IFileStorage : IDisposable -{ - Task ExistsAsync(string path, CancellationToken cancellationToken = default); - - Task> ListFilesAsync( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ); - - Task OpenReadAsync(string path, CancellationToken cancellationToken = default); - - Task OpenWriteAsync(string path, CancellationToken cancellationToken = default); - - Task GetDownloadUrlAsync(string path, DateTime expiresAt, CancellationToken cancellationToken = default); - - Task DeleteAsync(string path, bool recurse = false, CancellationToken cancellationToken = default); -} diff --git a/src/SIL.Machine.AspNetCore/Services/IFileSystem.cs b/src/SIL.Machine.AspNetCore/Services/IFileSystem.cs deleted file mode 100644 index 17f066028..000000000 --- a/src/SIL.Machine.AspNetCore/Services/IFileSystem.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface IFileSystem -{ - void DeleteFile(string path); - void CreateDirectory(string path); - Stream OpenWrite(string path); - Stream OpenRead(string path); -} diff --git a/src/SIL.Machine.AspNetCore/Services/IHangfireBuildJobFactory.cs b/src/SIL.Machine.AspNetCore/Services/IHangfireBuildJobFactory.cs deleted file mode 100644 index 3c1c37510..000000000 --- a/src/SIL.Machine.AspNetCore/Services/IHangfireBuildJobFactory.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface IHangfireBuildJobFactory -{ - TranslationEngineType EngineType { get; } - - Job CreateJob(string engineId, string buildId, BuildStage stage, object? data, string? buildOptions); -} diff --git a/src/SIL.Machine.AspNetCore/Services/ILanguageTagService.cs b/src/SIL.Machine.AspNetCore/Services/ILanguageTagService.cs deleted file mode 100644 index 95a65985f..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ILanguageTagService.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface ILanguageTagService -{ - bool ConvertToFlores200Code(string languageTag, out string flores200Code); -} diff --git a/src/SIL.Machine.AspNetCore/Services/IMessageOutboxService.cs b/src/SIL.Machine.AspNetCore/Services/IMessageOutboxService.cs deleted file mode 100644 index d2b0a975e..000000000 --- a/src/SIL.Machine.AspNetCore/Services/IMessageOutboxService.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface IMessageOutboxService -{ - public Task EnqueueMessageAsync( - string outboxId, - string method, - string groupId, - string? content = null, - Stream? contentStream = null, - CancellationToken cancellationToken = default - ); -} diff --git a/src/SIL.Machine.AspNetCore/Services/IOutboxMessageHandler.cs b/src/SIL.Machine.AspNetCore/Services/IOutboxMessageHandler.cs deleted file mode 100644 index 1d39a94d3..000000000 --- a/src/SIL.Machine.AspNetCore/Services/IOutboxMessageHandler.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface IOutboxMessageHandler -{ - public string OutboxId { get; } - - public Task HandleMessageAsync( - string method, - string? content, - Stream? contentStream, - CancellationToken cancellationToken = default - ); -} diff --git a/src/SIL.Machine.AspNetCore/Services/IPlatformService.cs b/src/SIL.Machine.AspNetCore/Services/IPlatformService.cs deleted file mode 100644 index 3d8f35c57..000000000 --- a/src/SIL.Machine.AspNetCore/Services/IPlatformService.cs +++ /dev/null @@ -1,30 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface IPlatformService -{ - Task IncrementTrainSizeAsync(string engineId, int count = 1, CancellationToken cancellationToken = default); - - Task UpdateBuildStatusAsync( - string buildId, - ProgressStatus progressStatus, - int? queueDepth = null, - CancellationToken cancellationToken = default - ); - Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken cancellationToken = default); - Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default); - Task BuildCompletedAsync( - string buildId, - int trainSize, - double confidence, - CancellationToken cancellationToken = default - ); - Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default); - Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default); - Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default); - - Task InsertPretranslationsAsync( - string engineId, - Stream pretranslationsStream, - CancellationToken cancellationToken = default - ); -} diff --git a/src/SIL.Machine.AspNetCore/Services/ISharedFileService.cs b/src/SIL.Machine.AspNetCore/Services/ISharedFileService.cs deleted file mode 100644 index f082a79c2..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ISharedFileService.cs +++ /dev/null @@ -1,24 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface ISharedFileService -{ - Uri GetBaseUri(); - - Uri GetResolvedUri(string path); - - Task GetDownloadUrlAsync(string path, DateTime expiresAt); - - Task> ListFilesAsync( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ); - - Task OpenReadAsync(string path, CancellationToken cancellationToken = default); - - Task OpenWriteAsync(string path, CancellationToken cancellationToken = default); - - Task ExistsAsync(string path, CancellationToken cancellationToken = default); - - Task DeleteAsync(string path, CancellationToken cancellationToken = default); -} diff --git a/src/SIL.Machine.AspNetCore/Services/ISmtModelFactory.cs b/src/SIL.Machine.AspNetCore/Services/ISmtModelFactory.cs deleted file mode 100644 index 292644766..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ISmtModelFactory.cs +++ /dev/null @@ -1,22 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface ISmtModelFactory -{ - Task CreateAsync( - string engineDir, - IRangeTokenizer tokenizer, - IDetokenizer detokenizer, - ITruecaser truecaser, - CancellationToken cancellationToken = default - ); - Task CreateTrainerAsync( - string engineDir, - IRangeTokenizer tokenizer, - IParallelTextCorpus corpus, - CancellationToken cancellationToken = default - ); - Task InitNewAsync(string engineDir, CancellationToken cancellationToken = default); - Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default); - Task UpdateEngineFromAsync(string engineDir, Stream source, CancellationToken cancellationToken = default); - Task SaveEngineToAsync(string engineDir, Stream destination, CancellationToken cancellationToken = default); -} diff --git a/src/SIL.Machine.AspNetCore/Services/ITransferEngineFactory.cs b/src/SIL.Machine.AspNetCore/Services/ITransferEngineFactory.cs deleted file mode 100644 index c99682e07..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ITransferEngineFactory.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface ITransferEngineFactory -{ - Task CreateAsync( - string engineDir, - IRangeTokenizer tokenizer, - IDetokenizer detokenizer, - ITruecaser truecaser, - CancellationToken cancellationToken = default - ); - Task InitNewAsync(string engineDir, CancellationToken cancellationToken = default); - Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default); -} diff --git a/src/SIL.Machine.AspNetCore/Services/ITranslationEngineService.cs b/src/SIL.Machine.AspNetCore/Services/ITranslationEngineService.cs deleted file mode 100644 index 0fbcdef03..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ITranslationEngineService.cs +++ /dev/null @@ -1,49 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface ITranslationEngineService -{ - TranslationEngineType Type { get; } - - Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - bool? isModelPersisted = null, - CancellationToken cancellationToken = default - ); - Task DeleteAsync(string engineId, CancellationToken cancellationToken = default); - - Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ); - - Task GetWordGraphAsync(string engineId, string segment, CancellationToken cancellationToken = default); - - Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ); - - Task StartBuildAsync( - string engineId, - string buildId, - string? buildOptions, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ); - - Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default); - - Task GetModelDownloadUrlAsync(string engineId, CancellationToken cancellationToken = default); - - int GetQueueSize(); - - bool IsLanguageNativeToModel(string language, out string internalCode); -} diff --git a/src/SIL.Machine.AspNetCore/Services/ITruecaserFactory.cs b/src/SIL.Machine.AspNetCore/Services/ITruecaserFactory.cs deleted file mode 100644 index 4395a7f29..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ITruecaserFactory.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public interface ITruecaserFactory -{ - Task CreateAsync(string engineDir, CancellationToken cancellationToken = default); - Task CreateTrainerAsync( - string engineDir, - ITokenizer tokenizer, - ITextCorpus corpus, - CancellationToken cancellationToken = default - ); - Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default); -} diff --git a/src/SIL.Machine.AspNetCore/Services/InMemoryStorage.cs b/src/SIL.Machine.AspNetCore/Services/InMemoryStorage.cs deleted file mode 100644 index 7deccb6e9..000000000 --- a/src/SIL.Machine.AspNetCore/Services/InMemoryStorage.cs +++ /dev/null @@ -1,142 +0,0 @@ -using SIL.ObjectModel; -using static SIL.Machine.AspNetCore.Utils.SharedFileUtils; - -namespace SIL.Machine.AspNetCore.Services; - -public class InMemoryStorage : DisposableBase, IFileStorage -{ - public class Entry : Stream - { - public MemoryStream MemoryStream { get; } - public string Path { get; } - - private readonly InMemoryStorage _parent; - - public override bool CanRead => MemoryStream.CanRead; - - public override bool CanSeek => MemoryStream.CanSeek; - - public override bool CanWrite => MemoryStream.CanWrite; - - public override long Length => MemoryStream.Length; - - public override long Position - { - get => MemoryStream.Position; - set => MemoryStream.Position = value; - } - - public Entry(string path, InMemoryStorage parent) - { - Path = path; - MemoryStream = new(); - _parent = parent; - } - - public Entry(Entry other) - { - Path = other.Path; - MemoryStream = other.MemoryStream; - _parent = other._parent; - } - - protected override void Dispose(bool disposing) - { - _parent._memoryStreams[Path] = new Entry(this); - } - - public override void Flush() - { - MemoryStream.Flush(); - } - - public override int Read(byte[] buffer, int offset, int count) - { - return MemoryStream.Read(buffer, offset, count); - } - - public override long Seek(long offset, SeekOrigin origin) - { - return MemoryStream.Seek(offset, origin); - } - - public override void SetLength(long value) - { - MemoryStream.SetLength(value); - } - - public override void Write(byte[] buffer, int offset, int count) - { - MemoryStream.Write(buffer, offset, count); - } - } - - private readonly ConcurrentDictionary _memoryStreams = new(); - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return Task.FromResult(_memoryStreams.TryGetValue(Normalize(path), out _)); - } - - public Task> ListFilesAsync( - string? path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - path = string.IsNullOrEmpty(path) ? "" : Normalize(path, includeTrailingSlash: true); - if (recurse) - { - return Task.FromResult>( - _memoryStreams.Keys.Where(p => p.StartsWith(path)).ToList() - ); - } - - return Task.FromResult>( - _memoryStreams.Keys.Where(p => p.StartsWith(path) && !p[path.Length..].Contains('/')).ToList() - ); - } - - public Task GetDownloadUrlAsync( - string path, - DateTime expiresAt, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - if (!_memoryStreams.TryGetValue(Normalize(path), out Entry? ret)) - throw new FileNotFoundException($"Unable to find file {path}"); - ret.Position = 0; - return Task.FromResult(ret); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return Task.FromResult(new Entry(Normalize(path), this)); - } - - public async Task DeleteAsync(string path, bool recurse, CancellationToken cancellationToken = default) - { - if (_memoryStreams.ContainsKey(Normalize(path))) - { - _memoryStreams.Remove(Normalize(path), out _); - } - else - { - IEnumerable filesToRemove = await ListFilesAsync(path, recurse, cancellationToken); - foreach (string filePath in filesToRemove) - _memoryStreams.Remove(Normalize(filePath), out _); - } - } - - protected override void DisposeManagedResources() - { - foreach (Entry stream in _memoryStreams.Values) - stream.Dispose(); - _memoryStreams.Clear(); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/LanguageTagService.cs b/src/SIL.Machine.AspNetCore/Services/LanguageTagService.cs deleted file mode 100644 index 72dca5135..000000000 --- a/src/SIL.Machine.AspNetCore/Services/LanguageTagService.cs +++ /dev/null @@ -1,170 +0,0 @@ -namespace SIL.Machine.Translation; - -public class LanguageTagService : ILanguageTagService -{ - private static readonly Dictionary StandardLanguages = - new() - { - { "ar", "arb" }, - { "ms", "zsm" }, - { "lv", "lvs" }, - { "ne", "npi" }, - { "sw", "swh" }, - { "cmn", "zh" } - }; - - private static readonly Dictionary StandardScripts = new() { { "Kore", "Hang" } }; - - private readonly Dictionary _defaultScripts; - - private readonly Dictionary _flores200Languages; - - private static readonly Regex LangTagPattern = - new("(?'language'[a-zA-Z]{2,8})([_-](?'script'[a-zA-Z]{4}))?", RegexOptions.ExplicitCapture); - - public LanguageTagService() - { - // initialize SLDR language tags to retrieve latest langtags.json file - _defaultScripts = InitializeDefaultScripts(); - _flores200Languages = InitializeFlores200Languages(); - } - - protected virtual void InitializeSldrLanguageTags() - { - Sldr.InitializeLanguageTags(); - } - - private Dictionary InitializeDefaultScripts() - { - InitializeSldrLanguageTags(); - var cachedAllTagsPath = Path.Combine(Sldr.SldrCachePath, "langtags.json"); - JsonNode? json; - - if (!File.Exists(cachedAllTagsPath)) - { - using HttpClient client = new(); - using HttpResponseMessage response = client.Send( - new HttpRequestMessage( - HttpMethod.Get, - "https://raw.githubusercontent.com/silnrsi/langtags/master/pub/langtags.json" - ) - ); - response.EnsureSuccessStatusCode(); - using Stream responseStream = response.Content.ReadAsStream(); - using FileStream fileStream = new(cachedAllTagsPath, FileMode.Create); - responseStream.CopyTo(fileStream); - } - using FileStream stream = new(cachedAllTagsPath, FileMode.Open); - json = JsonNode.Parse(stream); - - Dictionary tempDefaultScripts = new(); - foreach (JsonNode? entry in json!.AsArray()) - { - if (entry is null) - continue; - - var script = (string?)entry["script"]; - if (script is null) - continue; - - JsonNode? tags = entry["tags"]; - if (tags is not null) - { - foreach (var t in tags.AsArray().Select(v => (string?)v)) - { - if ( - t is not null - && IetfLanguageTag.TryGetParts(t, out _, out string? s, out _, out _) - && s is null - ) - { - tempDefaultScripts[t] = script; - } - } - } - - var tag = (string?)entry["tag"]; - if (tag is not null) - tempDefaultScripts[tag] = script; - } - return tempDefaultScripts; - } - - private static Dictionary InitializeFlores200Languages() - { - var tempFlores200Languages = new Dictionary(); - using var floresStream = Assembly - .GetExecutingAssembly() - .GetManifestResourceStream("SIL.Machine.AspNetCore.data.flores200languages.csv"); - Debug.Assert(floresStream is not null); - var reader = new StreamReader(floresStream); - var firstLine = reader.ReadLine(); - Debug.Assert(firstLine == "language, code"); - while (!reader.EndOfStream) - { - string? line = reader.ReadLine(); - if (line is null) - continue; - string[] values = line.Split(','); - tempFlores200Languages[values[1].Trim()] = values[0].Trim(); - } - return tempFlores200Languages; - } - - /** - * Converts a language tag to a Flores 200 code - * @param {string} languageTag - The language tag to convert - * @param out {string} flores200Code - The converted Flores 200 code - * @returns {bool} is the language is the Flores 200 list - */ - public bool ConvertToFlores200Code(string languageTag, out string flores200Code) - { - flores200Code = ResolveLanguageTag(languageTag); - return _flores200Languages.ContainsKey(flores200Code); - } - - private string ResolveLanguageTag(string languageTag) - { - // Try to find a pattern of {language code}_{script} - Match langTagMatch = LangTagPattern.Match(languageTag); - if (!langTagMatch.Success) - return languageTag; - string parsedLanguage = langTagMatch.Groups["language"].Value; - string languageSubtag = parsedLanguage; - string iso639_3Code = parsedLanguage; - - // Best attempt to convert language to a registered ISO 639-3 code - // Uses https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry for mapping - - // If they gave us the ISO code, revert it to the 2 character code - if (StandardSubtags.TryGetLanguageFromIso3Code(languageSubtag, out LanguageSubtag tempSubtag)) - languageSubtag = tempSubtag.Code; - - // There are a few extra conversions not in SIL Writing Systems that we need to handle - if (StandardLanguages.TryGetValue(languageSubtag, out string? tempName)) - languageSubtag = tempName; - - if (StandardSubtags.RegisteredLanguages.TryGet(languageSubtag, out LanguageSubtag? languageSubtagObj)) - iso639_3Code = languageSubtagObj.Iso3Code; - - // Use default script unless there is one parsed out of the language tag - Group scriptGroup = langTagMatch.Groups["script"]; - string? script = null; - - if (scriptGroup.Success) - script = scriptGroup.Value; - else if (_defaultScripts.TryGetValue(languageTag, out string? tempScript2)) - script = tempScript2; - else if (_defaultScripts.TryGetValue(languageSubtag, out string? tempScript)) - script = tempScript; - - // There are a few extra conversions not in SIL Writing Systems that we need to handle - if (script is not null && StandardScripts.TryGetValue(script, out string? tempScript3)) - script = tempScript3; - - if (script is not null) - return $"{iso639_3Code}_{script}"; - else - return languageTag; - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/LocalStorage.cs b/src/SIL.Machine.AspNetCore/Services/LocalStorage.cs deleted file mode 100644 index 38e9049bd..000000000 --- a/src/SIL.Machine.AspNetCore/Services/LocalStorage.cs +++ /dev/null @@ -1,78 +0,0 @@ -using SIL.ObjectModel; -using static SIL.Machine.AspNetCore.Utils.SharedFileUtils; - -namespace SIL.Machine.AspNetCore.Services; - -public class LocalStorage : DisposableBase, IFileStorage -{ - private readonly Uri _basePath; - - public LocalStorage(string basePath) - { - _basePath = new Uri(basePath); - if (!_basePath.AbsoluteUri.EndsWith("/")) - _basePath = new Uri(_basePath.AbsoluteUri + "/"); - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - Uri pathUri = new(_basePath, Normalize(path)); - return Task.FromResult(File.Exists(pathUri.LocalPath)); - } - - public Task> ListFilesAsync( - string path = "", - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - Uri pathUri = new(_basePath, Normalize(path)); - string[] files = Directory.GetFiles( - pathUri.LocalPath, - "*", - new EnumerationOptions { RecurseSubdirectories = recurse } - ); - return Task.FromResult>( - files.Select(f => _basePath.MakeRelativeUri(new Uri(f)).ToString()).ToArray() - ); - } - - public Task GetDownloadUrlAsync( - string path, - DateTime expiresAt, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - Uri pathUri = new(_basePath, Normalize(path)); - return Task.FromResult(File.OpenRead(pathUri.LocalPath)); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - Uri pathUri = new(_basePath, Normalize(path)); - Directory.CreateDirectory(Path.GetDirectoryName(pathUri.LocalPath)!); - return Task.FromResult(File.OpenWrite(pathUri.LocalPath)); - } - - public async Task DeleteAsync(string path, bool recurse, CancellationToken cancellationToken = default) - { - Uri pathUri = new(_basePath, Normalize(path)); - - if (File.Exists(pathUri.LocalPath)) - { - File.Delete(pathUri.LocalPath); - } - else if (Directory.Exists(pathUri.LocalPath)) - { - foreach (string filePath in await ListFilesAsync(path, recurse, cancellationToken)) - { - await DeleteAsync(filePath, false, cancellationToken); - } - } - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs b/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs deleted file mode 100644 index 092f66913..000000000 --- a/src/SIL.Machine.AspNetCore/Services/MessageOutboxDeliveryService.cs +++ /dev/null @@ -1,174 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class MessageOutboxDeliveryService( - IServiceProvider services, - IEnumerable outboxMessageHandlers, - IFileSystem fileSystem, - IOptionsMonitor options, - ILogger logger -) : BackgroundService -{ - private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(10); - - private readonly IServiceProvider _services = services; - private readonly Dictionary _outboxMessageHandlers = - outboxMessageHandlers.ToDictionary(o => o.OutboxId); - private readonly IFileSystem _fileSystem = fileSystem; - private readonly IOptionsMonitor _options = options; - private readonly ILogger _logger = logger; - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - Initialize(); - using IServiceScope scope = _services.CreateScope(); - var messages = scope.ServiceProvider.GetRequiredService>(); - using ISubscription subscription = await messages.SubscribeAsync(e => true, stoppingToken); - while (true) - { - await subscription.WaitForChangeAsync(timeout: Timeout, cancellationToken: stoppingToken); - if (stoppingToken.IsCancellationRequested) - break; - await ProcessMessagesAsync(messages, stoppingToken); - } - } - - private void Initialize() - { - _fileSystem.CreateDirectory(_options.CurrentValue.OutboxDir); - } - - internal async Task ProcessMessagesAsync( - IRepository messages, - CancellationToken cancellationToken = default - ) - { - bool anyMessages = await messages.ExistsAsync(m => true, cancellationToken); - if (!anyMessages) - return; - - IReadOnlyList curMessages = await messages.GetAllAsync(cancellationToken); - - IEnumerable> messageGroups = curMessages - .OrderBy(m => m.Index) - .GroupBy(m => (m.OutboxRef, m.GroupId)); - - foreach (IGrouping<(string OutboxId, string GroupId), OutboxMessage> messageGroup in messageGroups) - { - bool abortMessageGroup = false; - IOutboxMessageHandler outboxMessageHandler = _outboxMessageHandlers[messageGroup.Key.OutboxId]; - foreach (OutboxMessage message in messageGroup) - { - try - { - await ProcessGroupMessagesAsync(messages, message, outboxMessageHandler, cancellationToken); - } - catch (RpcException e) - { - switch (e.StatusCode) - { - case StatusCode.Unavailable: - case StatusCode.Unauthenticated: - case StatusCode.PermissionDenied: - case StatusCode.Cancelled: - _logger.LogWarning(e, "Platform Message sending failure: {statusCode}", e.StatusCode); - return; - case StatusCode.Aborted: - case StatusCode.DeadlineExceeded: - case StatusCode.Internal: - case StatusCode.ResourceExhausted: - case StatusCode.Unknown: - abortMessageGroup = !await CheckIfFinalMessageAttempt(messages, message, e); - break; - case StatusCode.InvalidArgument: - default: - // log error - await PermanentlyFailedMessage(messages, message, e); - break; - } - } - catch (Exception e) - { - await PermanentlyFailedMessage(messages, message, e); - break; - } - if (abortMessageGroup) - break; - } - } - } - - private async Task ProcessGroupMessagesAsync( - IRepository messages, - OutboxMessage message, - IOutboxMessageHandler outboxMessageHandler, - CancellationToken cancellationToken = default - ) - { - Stream? contentStream = null; - string filePath = Path.Combine(_options.CurrentValue.OutboxDir, message.Id); - if (message.HasContentStream) - contentStream = _fileSystem.OpenRead(filePath); - try - { - await outboxMessageHandler.HandleMessageAsync( - message.Method, - message.Content, - contentStream, - cancellationToken - ); - await messages.DeleteAsync(message.Id); - } - finally - { - contentStream?.Dispose(); - } - _fileSystem.DeleteFile(filePath); - } - - private async Task CheckIfFinalMessageAttempt( - IRepository messages, - OutboxMessage message, - Exception e - ) - { - if (message.Created < DateTimeOffset.UtcNow.Subtract(_options.CurrentValue.MessageExpirationTimeout)) - { - await PermanentlyFailedMessage(messages, message, e); - return true; - } - else - { - await LogFailedAttempt(messages, message, e); - return false; - } - } - - private async Task PermanentlyFailedMessage(IRepository messages, OutboxMessage message, Exception e) - { - // log error - _logger.LogError( - e, - "Permanently failed to process message {Id}: {Method} with content {Content} and error message: {ErrorMessage}", - message.Id, - message.Method, - message.Content, - e.Message - ); - await messages.DeleteAsync(message.Id); - } - - private async Task LogFailedAttempt(IRepository messages, OutboxMessage message, Exception e) - { - // log error - await messages.UpdateAsync(m => m.Id == message.Id, b => b.Inc(m => m.Attempts, 1)); - _logger.LogError( - e, - "Attempt {Attempts}. Failed to process message {Id}: {Method} with content {Content} and error message: {ErrorMessage}", - message.Attempts + 1, - message.Id, - message.Method, - message.Content, - e.Message - ); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs b/src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs deleted file mode 100644 index c69767d80..000000000 --- a/src/SIL.Machine.AspNetCore/Services/MessageOutboxService.cs +++ /dev/null @@ -1,75 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class MessageOutboxService( - IRepository outboxes, - IRepository messages, - IIdGenerator idGenerator, - IFileSystem fileSystem, - IOptionsMonitor options -) : IMessageOutboxService -{ - private readonly IRepository _outboxes = outboxes; - private readonly IRepository _messages = messages; - private readonly IIdGenerator _idGenerator = idGenerator; - private readonly IFileSystem _fileSystem = fileSystem; - private readonly IOptionsMonitor _options = options; - internal int MaxDocumentSize { get; set; } = 1_000_000; - - public async Task EnqueueMessageAsync( - string outboxId, - string method, - string groupId, - string? content = null, - Stream? contentStream = null, - CancellationToken cancellationToken = default - ) - { - if (content == null && contentStream == null) - { - throw new ArgumentException("Either content or contentStream must be specified."); - } - if (content is not null && content.Length > MaxDocumentSize) - { - throw new ArgumentException( - $"The content is too large for request {method} with group ID {groupId}. " - + $"It is {content.Length} bytes, but the maximum is {MaxDocumentSize} bytes." - ); - } - Outbox outbox = ( - await _outboxes.UpdateAsync( - outboxId, - u => u.Inc(o => o.CurrentIndex, 1), - upsert: true, - cancellationToken: cancellationToken - ) - )!; - OutboxMessage outboxMessage = - new() - { - Id = _idGenerator.GenerateId(), - Index = outbox.CurrentIndex, - OutboxRef = outboxId, - Method = method, - GroupId = groupId, - Content = content, - HasContentStream = contentStream is not null - }; - string filePath = Path.Combine(_options.CurrentValue.OutboxDir, outboxMessage.Id); - try - { - if (contentStream is not null) - { - await using Stream fileStream = _fileSystem.OpenWrite(filePath); - await contentStream.CopyToAsync(fileStream, cancellationToken); - } - await _messages.InsertAsync(outboxMessage, cancellationToken: cancellationToken); - return outboxMessage.Id; - } - catch - { - if (contentStream is not null) - _fileSystem.DeleteFile(filePath); - throw; - } - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/ModelCleanupService.cs b/src/SIL.Machine.AspNetCore/Services/ModelCleanupService.cs deleted file mode 100644 index f081a0f7a..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ModelCleanupService.cs +++ /dev/null @@ -1,59 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ModelCleanupService( - IServiceProvider services, - ISharedFileService sharedFileService, - ILogger logger -) : RecurrentTask("Model Cleanup Service", services, RefreshPeriod, logger) -{ - private readonly ISharedFileService _sharedFileService = sharedFileService; - private readonly ILogger _logger = logger; - private static readonly TimeSpan RefreshPeriod = TimeSpan.FromDays(1); - - protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken) - { - var engines = scope.ServiceProvider.GetRequiredService>(); - await CheckModelsAsync(engines, cancellationToken); - } - - internal async Task CheckModelsAsync(IRepository engines, CancellationToken cancellationToken) - { - _logger.LogInformation("Running model cleanup job"); - IReadOnlyCollection paths = await _sharedFileService.ListFilesAsync( - NmtEngineService.ModelDirectory, - cancellationToken: cancellationToken - ); - // Get all NMT engine ids from the database - IReadOnlyList? allEngines = await engines.GetAllAsync(cancellationToken: cancellationToken); - IEnumerable validNmtFilenames = allEngines - .Where(e => e.Type == TranslationEngineType.Nmt) - .Select(e => NmtEngineService.GetModelPath(e.EngineId, e.BuildRevision)); - // If there is a currently running build that creates and pushes a new file, but the database has not - // updated yet, don't delete the new file. - IEnumerable validNmtFilenamesForNextBuild = allEngines - .Where(e => e.Type == TranslationEngineType.Nmt) - .Select(e => NmtEngineService.GetModelPath(e.EngineId, e.BuildRevision + 1)); - - HashSet filenameFilter = validNmtFilenames.Concat(validNmtFilenamesForNextBuild).ToHashSet(); - - foreach (string path in paths) - { - if (!filenameFilter.Contains(path)) - { - await DeleteFileAsync( - path, - $"file in S3 bucket not found in database. It may be an old rev, etc.", - cancellationToken - ); - } - } - } - - private async Task DeleteFileAsync(string path, string message, CancellationToken cancellationToken = default) - { - // This may delete a file while it is being downloaded, but the chance is rare - // enough and the solution easy enough (just download again) to just live with it. - _logger.LogInformation("Deleting old model file {filename}: {message}", path, message); - await _sharedFileService.DeleteAsync(path, cancellationToken); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/NmtClearMLBuildJobFactory.cs b/src/SIL.Machine.AspNetCore/Services/NmtClearMLBuildJobFactory.cs deleted file mode 100644 index 5d507bba4..000000000 --- a/src/SIL.Machine.AspNetCore/Services/NmtClearMLBuildJobFactory.cs +++ /dev/null @@ -1,58 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class NmtClearMLBuildJobFactory( - ISharedFileService sharedFileService, - ILanguageTagService languageTagService, - IRepository engines -) : IClearMLBuildJobFactory -{ - private readonly ISharedFileService _sharedFileService = sharedFileService; - private readonly ILanguageTagService _languageTagService = languageTagService; - private readonly IRepository _engines = engines; - - public TranslationEngineType EngineType => TranslationEngineType.Nmt; - - public async Task CreateJobScriptAsync( - string engineId, - string buildId, - string modelType, - BuildStage stage, - object? data = null, - string? buildOptions = null, - CancellationToken cancellationToken = default - ) - { - if (stage == BuildStage.Train) - { - TranslationEngine? engine = await _engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException("The engine does not exist."); - - Uri sharedFileUri = _sharedFileService.GetBaseUri(); - string baseUri = sharedFileUri.GetComponents(UriComponents.SchemeAndServer, UriFormat.Unescaped); - string folder = sharedFileUri.GetComponents(UriComponents.Path, UriFormat.Unescaped); - _languageTagService.ConvertToFlores200Code(engine.SourceLanguage, out string srcLang); - _languageTagService.ConvertToFlores200Code(engine.TargetLanguage, out string trgLang); - return "from machine.jobs.build_nmt_engine import run\n" - + "args = {\n" - + $" 'model_type': '{modelType}',\n" - + $" 'engine_id': '{engineId}',\n" - + $" 'build_id': '{buildId}',\n" - + $" 'src_lang': '{srcLang}',\n" - + $" 'trg_lang': '{trgLang}',\n" - + $" 'shared_file_uri': '{baseUri}',\n" - + $" 'shared_file_folder': '{folder}',\n" - + (buildOptions is not null ? $" 'build_options': '''{buildOptions}''',\n" : "") - // buildRevision + 1 because the build revision is incremented after the build job - // is finished successfully but the file should be saved with the new revision number - + (engine.IsModelPersisted ? $" 'save_model': '{engineId}_{engine.BuildRevision + 1}',\n" : $"") - + $" 'clearml': True\n" - + "}\n" - + "run(args)\n"; - } - else - { - throw new ArgumentException("Unknown build stage.", nameof(stage)); - } - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/NmtEngineService.cs b/src/SIL.Machine.AspNetCore/Services/NmtEngineService.cs deleted file mode 100644 index 62d32e5d1..000000000 --- a/src/SIL.Machine.AspNetCore/Services/NmtEngineService.cs +++ /dev/null @@ -1,204 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class NmtEngineService( - IPlatformService platformService, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IRepository engines, - IBuildJobService buildJobService, - ILanguageTagService languageTagService, - IClearMLQueueService clearMLQueueService, - ISharedFileService sharedFileService -) : ITranslationEngineService -{ - private readonly IDistributedReaderWriterLockFactory _lockFactory = lockFactory; - private readonly IPlatformService _platformService = platformService; - private readonly IDataAccessContext _dataAccessContext = dataAccessContext; - private readonly IRepository _engines = engines; - private readonly IBuildJobService _buildJobService = buildJobService; - private readonly IClearMLQueueService _clearMLQueueService = clearMLQueueService; - private readonly ILanguageTagService _languageTagService = languageTagService; - private readonly ISharedFileService _sharedFileService = sharedFileService; - public const string ModelDirectory = "models/"; - - public static string GetModelPath(string engineId, int buildRevision) - { - return $"{ModelDirectory}{engineId}_{buildRevision}.tar.gz"; - } - - public TranslationEngineType Type => TranslationEngineType.Nmt; - - private const int MinutesToExpire = 60; - - public async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - bool? isModelPersisted = null, - CancellationToken cancellationToken = default - ) - { - var translationEngine = await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - var translationEngine = new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage, - Type = TranslationEngineType.Nmt, - IsModelPersisted = isModelPersisted ?? false // models are not persisted if not specified - }; - await _engines.InsertAsync(translationEngine, ct); - await _buildJobService.CreateEngineAsync(engineId, engineName, ct); - return translationEngine; - }, - cancellationToken: cancellationToken - ); - return translationEngine; - } - - public async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildJobAsync(engineId, cancellationToken); - - await _engines.DeleteAsync(e => e.EngineId == engineId, cancellationToken); - await _buildJobService.DeleteEngineAsync(engineId, CancellationToken.None); - } - await _lockFactory.DeleteAsync(engineId, CancellationToken.None); - } - - public async Task StartBuildAsync( - string engineId, - string buildId, - string? buildOptions, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - // If there is a pending/running build, then no need to start a new one. - if (await _buildJobService.IsEngineBuilding(engineId, cancellationToken)) - throw new InvalidOperationException("The engine is already building or in the process of canceling."); - - await _buildJobService.StartBuildJobAsync( - BuildJobRunnerType.Hangfire, - engineId, - buildId, - BuildStage.Preprocess, - corpora, - buildOptions, - cancellationToken - ); - } - } - - public async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - if (!await CancelBuildJobAsync(engineId, cancellationToken)) - throw new InvalidOperationException("The engine is not currently building."); - } - } - - public async Task GetModelDownloadUrlAsync( - string engineId, - CancellationToken cancellationToken = default - ) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.IsModelPersisted != true) - { - throw new NotSupportedException( - "The model cannot be downloaded. " - + "To enable downloading the model, recreate the engine with IsModelPersisted property to true." - ); - } - - if (engine.BuildRevision == 0) - throw new InvalidOperationException("The engine has not been built yet."); - string filepath = GetModelPath(engineId, engine.BuildRevision); - bool fileExists = await _sharedFileService.ExistsAsync(filepath, cancellationToken); - if (!fileExists) - throw new FileNotFoundException($"The model for build revision , {engine.BuildRevision}, does not exist."); - var expiresAt = DateTime.UtcNow.AddMinutes(MinutesToExpire); - var modelInfo = new ModelDownloadUrl - { - Url = await _sharedFileService.GetDownloadUrlAsync(filepath, expiresAt), - ModelRevision = engine.BuildRevision, - ExpiresAt = expiresAt - }; - return modelInfo; - } - - public Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - public int GetQueueSize() - { - return _clearMLQueueService.GetQueueSize(Type); - } - - public bool IsLanguageNativeToModel(string language, out string internalCode) - { - return _languageTagService.ConvertToFlores200Code(language, out internalCode); - } - - private async Task CancelBuildJobAsync(string engineId, CancellationToken cancellationToken) - { - string? buildId = null; - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - (buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync(engineId, ct); - if (buildId is not null && jobState is BuildJobState.None) - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - }, - cancellationToken: cancellationToken - ); - return buildId is not null; - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await _engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException($"The engine {engineId} does not exist."); - return engine; - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/NmtHangfireBuildJobFactory.cs b/src/SIL.Machine.AspNetCore/Services/NmtHangfireBuildJobFactory.cs deleted file mode 100644 index c0366078c..000000000 --- a/src/SIL.Machine.AspNetCore/Services/NmtHangfireBuildJobFactory.cs +++ /dev/null @@ -1,26 +0,0 @@ -using static SIL.Machine.AspNetCore.Services.HangfireBuildJobRunner; - -namespace SIL.Machine.AspNetCore.Services; - -public class NmtHangfireBuildJobFactory : IHangfireBuildJobFactory -{ - public TranslationEngineType EngineType => TranslationEngineType.Nmt; - - public Job CreateJob(string engineId, string buildId, BuildStage stage, object? data, string? buildOptions) - { - return stage switch - { - BuildStage.Preprocess - => CreateJob>( - engineId, - buildId, - "nmt", - data, - buildOptions - ), - BuildStage.Postprocess - => CreateJob(engineId, buildId, "nmt", data, buildOptions), - _ => throw new ArgumentException("Unknown build stage.", nameof(stage)), - }; - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/NmtPreprocessBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/NmtPreprocessBuildJob.cs deleted file mode 100644 index 64c51589a..000000000 --- a/src/SIL.Machine.AspNetCore/Services/NmtPreprocessBuildJob.cs +++ /dev/null @@ -1,31 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class NmtPreprocessBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - ILogger logger, - IBuildJobService buildJobService, - ISharedFileService sharedFileService, - ICorpusService corpusService, - ILanguageTagService languageTagService -) - : PreprocessBuildJob( - platformService, - engines, - lockFactory, - dataAccessContext, - logger, - buildJobService, - sharedFileService, - corpusService - ) -{ - private readonly ILanguageTagService _languageTagService = languageTagService; - - protected override bool ResolveLanguageCodeForBaseModel(string languageCode, out string resolvedCode) - { - return _languageTagService.ConvertToFlores200Code(languageCode, out resolvedCode); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs deleted file mode 100644 index dd1d7b387..000000000 --- a/src/SIL.Machine.AspNetCore/Services/PostprocessBuildJob.cs +++ /dev/null @@ -1,88 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class PostprocessBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, - ILogger logger, - ISharedFileService sharedFileService -) : HangfireBuildJob<(int, double)>(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) -{ - protected ISharedFileService SharedFileService { get; } = sharedFileService; - - protected override async Task DoWorkAsync( - string engineId, - string buildId, - (int, double) data, - string? buildOptions, - IDistributedReaderWriterLock @lock, - CancellationToken cancellationToken - ) - { - (int corpusSize, double confidence) = data; - - await using ( - Stream pretranslationsStream = await SharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ) - ) - { - await PlatformService.InsertPretranslationsAsync(engineId, pretranslationsStream, cancellationToken); - } - - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - await DataAccessContext.WithTransactionAsync( - async (ct) => - { - int additionalCorpusSize = await SaveModelAsync(engineId, buildId); - await PlatformService.BuildCompletedAsync( - buildId, - corpusSize + additionalCorpusSize, - Math.Round(confidence, 2, MidpointRounding.AwayFromZero), - CancellationToken.None - ); - await BuildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: true, - CancellationToken.None - ); - }, - cancellationToken: CancellationToken.None - ); - } - - Logger.LogInformation("Build completed ({0}).", buildId); - } - - protected virtual Task SaveModelAsync(string engineId, string buildId) - { - return Task.FromResult(0); - } - - protected override async Task CleanupAsync( - string engineId, - string buildId, - (int, double) data, - IDistributedReaderWriterLock @lock, - JobCompletionStatus completionStatus - ) - { - if (completionStatus is JobCompletionStatus.Restarting) - return; - - try - { - if (completionStatus is not JobCompletionStatus.Faulted) - await SharedFileService.DeleteAsync($"builds/{buildId}/"); - } - catch (Exception e) - { - Logger.LogWarning(e, "Unable to to delete job data for build {0}.", buildId); - } - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/PreprocessBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/PreprocessBuildJob.cs deleted file mode 100644 index 3f07ecbe1..000000000 --- a/src/SIL.Machine.AspNetCore/Services/PreprocessBuildJob.cs +++ /dev/null @@ -1,440 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class PreprocessBuildJob : HangfireBuildJob> -{ - private static readonly JsonWriterOptions PretranslateWriterOptions = new() { Indented = true }; - - internal BuildJobRunnerType TrainJobRunnerType { get; init; } = BuildJobRunnerType.ClearML; - - private readonly ISharedFileService _sharedFileService; - private readonly ICorpusService _corpusService; - private int _seed = 1234; - private Random _random; - - public PreprocessBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - ILogger logger, - IBuildJobService buildJobService, - ISharedFileService sharedFileService, - ICorpusService corpusService - ) - : base(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) - { - _sharedFileService = sharedFileService; - _corpusService = corpusService; - _random = new Random(_seed); - } - - internal int Seed - { - get => _seed; - set - { - if (_seed != value) - { - _seed = value; - _random = new Random(_seed); - } - } - } - - protected override async Task DoWorkAsync( - string engineId, - string buildId, - IReadOnlyList data, - string? buildOptions, - IDistributedReaderWriterLock @lock, - CancellationToken cancellationToken - ) - { - (int trainCount, int pretranslateCount) = await WriteDataFilesAsync( - buildId, - data, - buildOptions, - cancellationToken - ); - - // Log summary of build data - JsonObject buildPreprocessSummary = - new() - { - { "Event", "BuildPreprocess" }, - { "EngineId", engineId }, - { "BuildId", buildId }, - { "NumTrainRows", trainCount }, - { "NumPretranslateRows", pretranslateCount } - }; - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled."); - - bool sourceTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.SourceLanguage, out string srcLang); - buildPreprocessSummary.Add("SourceLanguageResolved", srcLang); - bool targetTagInBaseModel = ResolveLanguageCodeForBaseModel(engine.TargetLanguage, out string trgLang); - buildPreprocessSummary.Add("TargetLanguageResolved", trgLang); - Logger.LogInformation("{summary}", buildPreprocessSummary.ToJsonString()); - - if (trainCount == 0 && (!sourceTagInBaseModel || !targetTagInBaseModel)) - { - throw new InvalidOperationException( - $"Neither language code in build {buildId} are known to the base model, and the data specified for training was empty. Build canceled." - ); - } - - cancellationToken.ThrowIfCancellationRequested(); - - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - bool canceling = !await BuildJobService.StartBuildJobAsync( - TrainJobRunnerType, - engineId, - buildId, - BuildStage.Train, - buildOptions: buildOptions, - cancellationToken: cancellationToken - ); - if (canceling) - throw new OperationCanceledException(); - } - } - - private async Task<(int TrainCount, int PretranslateCount)> WriteDataFilesAsync( - string buildId, - IReadOnlyList corpora, - string? buildOptions, - CancellationToken cancellationToken - ) - { - JsonObject? buildOptionsObject = null; - if (buildOptions is not null) - { - buildOptionsObject = JsonSerializer.Deserialize(buildOptions); - } - await using StreamWriter sourceTrainWriter = - new(await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.src.txt", cancellationToken)); - await using StreamWriter targetTrainWriter = - new(await _sharedFileService.OpenWriteAsync($"builds/{buildId}/train.trg.txt", cancellationToken)); - - await using Stream pretranslateStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - await using Utf8JsonWriter pretranslateWriter = new(pretranslateStream, PretranslateWriterOptions); - - int trainCount = 0; - int pretranslateCount = 0; - pretranslateWriter.WriteStartArray(); - foreach (Corpus corpus in corpora) - { - ITextCorpus[] sourceTextCorpora = _corpusService.CreateTextCorpora(corpus.SourceFiles).ToArray(); - ITextCorpus targetTextCorpus = - _corpusService.CreateTextCorpora(corpus.TargetFiles).FirstOrDefault() ?? new DictionaryTextCorpus(); - - if (sourceTextCorpora.Length == 0) - continue; - - int skipCount = 0; - foreach (Row?[] rows in AlignTrainCorpus(sourceTextCorpora, targetTextCorpus)) - { - if (skipCount > 0) - { - skipCount--; - continue; - } - - Row[] trainRows = rows.Where(r => r is not null && IsInTrain(r, corpus)).Cast().ToArray(); - if (trainRows.Length > 0) - { - Row row = trainRows[0]; - if (rows.Length > 1) - { - Row[] nonEmptyRows = trainRows.Where(r => r.SourceSegment.Length > 0).ToArray(); - if (nonEmptyRows.Length > 0) - row = nonEmptyRows[_random.Next(nonEmptyRows.Length)]; - } - - await sourceTrainWriter.WriteAsync($"{row.SourceSegment}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetSegment}\n"); - skipCount = row.RowCount - 1; - if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) - trainCount++; - } - } - - if ((bool?)buildOptionsObject?["use_key_terms"] ?? true) - { - ITextCorpus? sourceTermCorpus = _corpusService.CreateTermCorpora(corpus.SourceFiles).FirstOrDefault(); - ITextCorpus? targetTermCorpus = _corpusService.CreateTermCorpora(corpus.TargetFiles).FirstOrDefault(); - if (sourceTermCorpus is not null && targetTermCorpus is not null) - { - IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpus.AlignRows(targetTermCorpus); - foreach (ParallelTextRow row in parallelKeyTermsCorpus) - { - await sourceTrainWriter.WriteAsync($"{row.SourceText}\n"); - await targetTrainWriter.WriteAsync($"{row.TargetText}\n"); - trainCount++; - } - } - } - - foreach (Row row in AlignPretranslateCorpus(sourceTextCorpora[0], targetTextCorpus)) - { - if ( - IsInPretranslate(row, corpus) - && row.SourceSegment.Length > 0 - && (row.TargetSegment.Length == 0 || !IsInTrain(row, corpus)) - ) - { - pretranslateWriter.WriteStartObject(); - pretranslateWriter.WriteString("corpusId", corpus.Id); - pretranslateWriter.WriteString("textId", row.TextId); - pretranslateWriter.WriteStartArray("refs"); - foreach (object rowRef in row.Refs) - pretranslateWriter.WriteStringValue(rowRef.ToString()); - pretranslateWriter.WriteEndArray(); - pretranslateWriter.WriteString("translation", row.SourceSegment); - pretranslateWriter.WriteEndObject(); - pretranslateCount++; - } - } - } - - pretranslateWriter.WriteEndArray(); - - return (trainCount, pretranslateCount); - } - - protected override async Task CleanupAsync( - string engineId, - string buildId, - IReadOnlyList data, - IDistributedReaderWriterLock @lock, - JobCompletionStatus completionStatus - ) - { - if (completionStatus is JobCompletionStatus.Canceled) - { - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/"); - } - catch (Exception e) - { - Logger.LogWarning(e, "Unable to to delete job data for build {BuildId}.", buildId); - } - } - } - - private static bool IsInTrain(Row row, Corpus corpus) - { - return IsIncluded(row, corpus.TrainOnTextIds, corpus.TrainOnChapters); - } - - private static bool IsInPretranslate(Row row, Corpus corpus) - { - return IsIncluded(row, corpus.PretranslateTextIds, corpus.PretranslateChapters); - } - - private static bool IsIncluded( - Row? row, - IReadOnlySet? textIds, - IReadOnlyDictionary>? chapters - ) - { - if (row is null) - return false; - if (chapters is not null) - { - return row.Refs.Any(r => IsInChapters(chapters, r)); - } - if (textIds is not null) - { - return textIds.Contains(row.TextId); - } - return true; - } - - private static bool IsInChapters(IReadOnlyDictionary> bookChapters, object rowRef) - { - if (rowRef is not ScriptureRef sr) - return false; - return bookChapters.TryGetValue(sr.Book, out HashSet? chapters) - && (chapters.Contains(sr.ChapterNum) || chapters.Count == 0); - } - - private static IEnumerable AlignTrainCorpus(IReadOnlyList srcCorpora, ITextCorpus trgCorpus) - { - if (trgCorpus.IsScripture()) - { - return srcCorpora - .Select(sc => AlignScripture(sc, trgCorpus)) - .ZipMany(rows => rows.ToArray()) - // filter out every list that only contains completely empty rows - .Where(rows => rows.Any(r => r is null || r.SourceSegment.Length > 0 || r.TargetSegment.Length > 0)); - } - - IEnumerable sourceOnlyRows = srcCorpora - .Select(sc => sc.AlignRows(trgCorpus, allSourceRows: true)) - .ZipMany(rows => - rows.Where(r => r.TargetSegment.Count == 0) - .Select(r => new Row(r.TextId, r.Refs, r.SourceText, r.TargetText, 1)) - .ToArray() - ); - - IEnumerable targetRows = srcCorpora - .Select(sc => sc.AlignRows(trgCorpus, allTargetRows: true)) - .ZipMany(rows => - rows.Where(r => r.TargetSegment.Count > 0) - .Select(r => new Row(r.TextId, r.Refs, r.SourceText, r.TargetText, 1)) - .ToArray() - ); - - return sourceOnlyRows - .Concat(targetRows) - // filter out every list that only contains completely empty rows - .Where(rows => rows.Any(r => r.SourceSegment.Length > 0 || r.TargetSegment.Length > 0)); - } - - private static IEnumerable AlignScripture(ITextCorpus srcCorpus, ITextCorpus trgCorpus) - { - int rowCount = 0; - StringBuilder srcSegBuffer = new(); - StringBuilder trgSegBuffer = new(); - HashSet vrefs = []; - foreach ( - (VerseRef vref, string srcSegment, string trgSegment) in srcCorpus - .ExtractScripture() - .Select(r => (r.CorpusVerseRef, r.Text)) - .Zip( - trgCorpus.ExtractScripture().Select(r => r.Text), - (s, t) => (VerseRef: s.CorpusVerseRef, SourceSegment: s.Text, TargetSegment: t) - ) - ) - { - if (srcSegment == "" && trgSegment == "") - { - vrefs.UnionWith(vref.AllVerses()); - rowCount++; - } - else if (srcSegment == "") - { - vrefs.UnionWith(vref.AllVerses()); - if (trgSegment.Length > 0) - { - if (trgSegBuffer.Length > 0) - trgSegBuffer.Append(' '); - trgSegBuffer.Append(trgSegment); - } - rowCount++; - } - else if (trgSegment == "") - { - vrefs.UnionWith(vref.AllVerses()); - if (srcSegment.Length > 0) - { - if (srcSegBuffer.Length > 0) - srcSegBuffer.Append(' '); - srcSegBuffer.Append(srcSegment); - } - rowCount++; - } - else - { - if (rowCount > 0) - { - yield return new( - vrefs.First().Book, - vrefs.Order().Select(v => new ScriptureRef(v)).Cast().ToArray(), - srcSegBuffer.ToString(), - trgSegBuffer.ToString(), - rowCount - ); - for (int i = 0; i < rowCount - 1; i++) - yield return null; - srcSegBuffer.Clear(); - trgSegBuffer.Clear(); - vrefs.Clear(); - rowCount = 0; - } - vrefs.UnionWith(vref.AllVerses()); - srcSegBuffer.Append(srcSegment); - trgSegBuffer.Append(trgSegment); - rowCount++; - } - } - - if (rowCount > 0) - { - yield return new( - vrefs.First().Book, - vrefs.Order().Select(v => new ScriptureRef(v)).Cast().ToArray(), - srcSegBuffer.ToString(), - trgSegBuffer.ToString(), - rowCount - ); - for (int i = 0; i < rowCount - 1; i++) - yield return null; - } - } - - private static IEnumerable AlignPretranslateCorpus(ITextCorpus srcCorpus, ITextCorpus trgCorpus) - { - int rowCount = 0; - StringBuilder srcSegBuffer = new(); - StringBuilder trgSegBuffer = new(); - List refs = []; - string textId = ""; - foreach (ParallelTextRow row in srcCorpus.AlignRows(trgCorpus, allSourceRows: true)) - { - if (!row.IsTargetRangeStart && row.IsTargetInRange) - { - refs.AddRange(row.Refs); - if (row.SourceText.Length > 0) - { - if (srcSegBuffer.Length > 0) - srcSegBuffer.Append(' '); - srcSegBuffer.Append(row.SourceText); - } - rowCount++; - } - else - { - if (rowCount > 0) - { - yield return new(textId, refs, srcSegBuffer.ToString(), trgSegBuffer.ToString(), 1); - textId = ""; - srcSegBuffer.Clear(); - trgSegBuffer.Clear(); - refs.Clear(); - rowCount = 0; - } - - textId = row.TextId; - refs.AddRange(row.Refs); - srcSegBuffer.Append(row.SourceText); - trgSegBuffer.Append(row.TargetText); - rowCount++; - } - } - - if (rowCount > 0) - yield return new(textId, refs, srcSegBuffer.ToString(), trgSegBuffer.ToString(), 1); - } - - private record Row( - string TextId, - IReadOnlyList Refs, - string SourceSegment, - string TargetSegment, - int RowCount - ); - - protected virtual bool ResolveLanguageCodeForBaseModel(string languageCode, out string resolvedCode) - { - resolvedCode = languageCode; - return true; - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/S3FileStorage.cs b/src/SIL.Machine.AspNetCore/Services/S3FileStorage.cs deleted file mode 100644 index a9d265e9c..000000000 --- a/src/SIL.Machine.AspNetCore/Services/S3FileStorage.cs +++ /dev/null @@ -1,125 +0,0 @@ -using SIL.ObjectModel; -using static SIL.Machine.AspNetCore.Utils.SharedFileUtils; - -namespace SIL.Machine.AspNetCore.Services; - -public class S3FileStorage : DisposableBase, IFileStorage -{ - private readonly AmazonS3Client _client; - private readonly string _bucketName; - private readonly string _basePath; - private readonly ILoggerFactory _loggerFactory; - - public S3FileStorage( - string bucketName, - string basePath, - string accessKeyId, - string secretAccessKey, - string region, - ILoggerFactory loggerFactory - ) - { - _client = new AmazonS3Client( - accessKeyId, - secretAccessKey, - new AmazonS3Config { RegionEndpoint = RegionEndpoint.GetBySystemName(region) } - ); - - _bucketName = bucketName; - // Ultimately, object keys can neither begin nor end with slashes; this is what broke the earlier low-level - // implementation - _basePath = Normalize(basePath, includeTrailingSlash: true); - _loggerFactory = loggerFactory; - } - - public async Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + Normalize(path), - MaxKeys = 1 - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - - return response.S3Objects.Any(); - } - - public async Task> ListFilesAsync( - string? path = null, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - if (path != null && !path.EndsWith("/")) - throw new ArgumentException("Path must be a folder (ending with '/')", nameof(path)); - - var request = new ListObjectsV2Request - { - BucketName = _bucketName, - Prefix = _basePath + (string.IsNullOrEmpty(path) ? "" : Normalize(path, includeTrailingSlash: true)), - Delimiter = recurse ? "" : "/" - }; - - ListObjectsV2Response response = await _client.ListObjectsV2Async(request, cancellationToken); - return response.S3Objects.Select(s3Obj => s3Obj.Key[_basePath.Length..]).ToList(); - } - - public Task GetDownloadUrlAsync( - string path, - DateTime expiresAt, - CancellationToken cancellationToken = default - ) - { - return Task.FromResult( - _client.GetPreSignedURL( - new GetPreSignedUrlRequest - { - BucketName = _bucketName, - Key = _basePath + Normalize(path), - Expires = expiresAt, - ResponseHeaderOverrides = new ResponseHeaderOverrides - { - ContentDisposition = new ContentDisposition() { FileName = Path.GetFileName(path) }.ToString() - } - } - ) - ); - } - - public async Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - GetObjectRequest request = new() { BucketName = _bucketName, Key = _basePath + Normalize(path) }; - GetObjectResponse response = await _client.GetObjectAsync(request, cancellationToken); - if (response.HttpStatusCode != HttpStatusCode.OK) - throw new FileNotFoundException($"File {path} does not exist"); - return response.ResponseStream; - } - - public async Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - string fullPath = _basePath + Normalize(path); - InitiateMultipartUploadRequest request = new() { BucketName = _bucketName, Key = fullPath }; - InitiateMultipartUploadResponse response = await _client.InitiateMultipartUploadAsync( - request, - cancellationToken - ); - return new BufferedStream( - new S3WriteStream(_client, fullPath, _bucketName, response.UploadId, _loggerFactory), - S3WriteStream.MaxPartSize - ); - } - - public async Task DeleteAsync(string path, bool recurse = false, CancellationToken cancellationToken = default) - { - DeleteObjectRequest request = new() { BucketName = _bucketName, Key = _basePath + Normalize(path) }; - DeleteObjectResponse response = await _client.DeleteObjectAsync(request, cancellationToken); - if (!response.HttpStatusCode.Equals(HttpStatusCode.NoContent)) - { - throw new HttpRequestException( - $"Received status code {response.HttpStatusCode} when attempting to delete {path}" - ); - } - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/S3HealthCheck.cs b/src/SIL.Machine.AspNetCore/Services/S3HealthCheck.cs deleted file mode 100644 index 5f5760cc2..000000000 --- a/src/SIL.Machine.AspNetCore/Services/S3HealthCheck.cs +++ /dev/null @@ -1,67 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3HealthCheck(IOptions options) : IHealthCheck -{ - private readonly IOptions _options = options; - private int _numConsecutiveFailures = 0; - private readonly AsyncLock _lock = new AsyncLock(); - - public async Task CheckHealthAsync( - HealthCheckContext context, - CancellationToken cancellationToken = default - ) - { - try - { - var request = new ListObjectsV2Request - { - BucketName = new Uri(_options.Value.Uri).Host, - Prefix = new Uri(_options.Value.Uri).AbsolutePath + "/models/", - MaxKeys = 1, - Delimiter = "" - }; - - await ( - new AmazonS3Client( - _options.Value.S3AccessKeyId, - _options.Value.S3SecretAccessKey, - new AmazonS3Config - { - MaxErrorRetry = 0, //Do not let health check hang - RegionEndpoint = RegionEndpoint.GetBySystemName(_options.Value.S3Region) - } - ) - ).ListObjectsV2Async(request, cancellationToken); - using (await _lock.LockAsync()) - _numConsecutiveFailures = 0; - return HealthCheckResult.Healthy("The S3 bucket is available"); - } - catch (Exception e) - { - using (await _lock.LockAsync()) - { - _numConsecutiveFailures++; - if ( - e is HttpRequestException httpRequestException - && httpRequestException.StatusCode is HttpStatusCode.Forbidden or HttpStatusCode.Unauthorized - ) - { - return _numConsecutiveFailures > 3 - ? HealthCheckResult.Unhealthy( - "S3 bucket is not available because of an authentication error. Please verify that credentials are valid." - ) - : HealthCheckResult.Degraded( - "S3 bucket is not available because of an authentication error. Please verify that credentials are valid." - ); - } - return _numConsecutiveFailures > 3 - ? HealthCheckResult.Unhealthy( - "S3 bucket is not available. The following exception occurred: " + e.Message - ) - : HealthCheckResult.Degraded( - "S3 bucket is not available. The following exception occurred: " + e.Message - ); - } - } - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/S3WriteStream.cs b/src/SIL.Machine.AspNetCore/Services/S3WriteStream.cs deleted file mode 100644 index 35130134f..000000000 --- a/src/SIL.Machine.AspNetCore/Services/S3WriteStream.cs +++ /dev/null @@ -1,222 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class S3WriteStream( - AmazonS3Client client, - string key, - string bucketName, - string uploadId, - ILoggerFactory loggerFactory -) : Stream -{ - private readonly AmazonS3Client _client = client; - private readonly string _key = key; - private readonly string _uploadId = uploadId; - private readonly string _bucketName = bucketName; - private readonly List _uploadResponses = new List(); - private readonly ILogger _logger = loggerFactory.CreateLogger(); - - public const int MaxPartSize = 5 * 1024 * 1024; - - public override bool CanRead => false; - - public override bool CanSeek => false; - - public override bool CanWrite => true; - - public override long Length => 0; - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); - } - - public override void Flush() { } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - - public override void SetLength(long value) => throw new NotSupportedException(); - - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override async ValueTask WriteAsync( - ReadOnlyMemory buffer, - CancellationToken cancellationToken = default - ) - { - try - { - using Stream stream = buffer.AsStream(); - - int bytesWritten = 0; - - while (stream.Length > bytesWritten) - { - int partNumber = _uploadResponses.Count + 1; - UploadPartRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId, - PartNumber = partNumber, - InputStream = stream, - PartSize = MaxPartSize - }; - request.StreamTransferProgress += new EventHandler( - (_, e) => - { - _logger.LogDebug( - "Transferred {e.TransferredBytes}/{e.TotalBytes}", - e.TransferredBytes, - e.TotalBytes - ); - } - ); - UploadPartResponse response = await _client.UploadPartAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - { - throw new HttpRequestException( - $"Tried to upload part {partNumber} of upload {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - - _uploadResponses.Add(response); - - bytesWritten += MaxPartSize; - } - } - catch (Exception e) - { - await AbortAsync(e); - throw; - } - } - - public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - await WriteAsync(buffer.AsMemory(offset, count), cancellationToken); - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - if (_uploadResponses.Count == 0) - { - AbortAsync().WaitAndUnwrapException(); - PutObjectRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - ContentBody = "" - }; - PutObjectResponse response = _client.PutObjectAsync(request).WaitAndUnwrapException(); - if (response.HttpStatusCode != HttpStatusCode.OK) - { - throw new HttpRequestException( - $"Tried to upload empty file to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - } - else - { - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = _client - .CompleteMultipartUploadAsync(request) - .WaitAndUnwrapException(); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - { - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - } - catch (Exception e) - { - AbortAsync(e).WaitAndUnwrapException(); - throw; - } - } - } - base.Dispose(disposing); - } - - public override async ValueTask DisposeAsync() - { - if (_uploadResponses.Count == 0) - { - await AbortAsync(); - PutObjectRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - ContentBody = "" - }; - PutObjectResponse response = await _client.PutObjectAsync(request); - if (response.HttpStatusCode != HttpStatusCode.OK) - { - throw new HttpRequestException( - $"Tried to upload empty file to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - - return; - } - try - { - CompleteMultipartUploadRequest request = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - request.AddPartETags(_uploadResponses); - CompleteMultipartUploadResponse response = await _client.CompleteMultipartUploadAsync(request); - Dispose(disposing: false); - GC.SuppressFinalize(this); - if (response.HttpStatusCode != HttpStatusCode.OK) - { - throw new HttpRequestException( - $"Tried to complete {_uploadId} to {_bucketName}/{_key} but received response code {response.HttpStatusCode}" - ); - } - } - catch (Exception e) - { - await AbortAsync(e); - } - } - - private async Task AbortAsync(Exception? e = null) - { - if (e is not null) - _logger.LogError(e, $"Aborted upload {_uploadId} to {_bucketName}/{_key}"); - AbortMultipartUploadRequest abortMPURequest = - new() - { - BucketName = _bucketName, - Key = _key, - UploadId = _uploadId - }; - await _client.AbortMultipartUploadAsync(abortMPURequest); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxConstants.cs b/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxConstants.cs deleted file mode 100644 index 39e6656ad..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxConstants.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public static class ServalPlatformOutboxConstants -{ - public const string OutboxId = "ServalPlatform"; - - public const string BuildStarted = "BuildStarted"; - public const string BuildCompleted = "BuildCompleted"; - public const string BuildCanceled = "BuildCanceled"; - public const string BuildFaulted = "BuildFaulted"; - public const string BuildRestarting = "BuildRestarting"; - public const string InsertPretranslations = "InsertPretranslations"; - public const string IncrementTranslationEngineCorpusSize = "IncrementTranslationEngineCorpusSize"; -} diff --git a/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxMessageHandler.cs b/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxMessageHandler.cs deleted file mode 100644 index 9c32ff0e0..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ServalPlatformOutboxMessageHandler.cs +++ /dev/null @@ -1,92 +0,0 @@ -using Serval.Translation.V1; - -namespace SIL.Machine.AspNetCore.Services; - -public class ServalPlatformOutboxMessageHandler(TranslationPlatformApi.TranslationPlatformApiClient client) - : IOutboxMessageHandler -{ - private readonly TranslationPlatformApi.TranslationPlatformApiClient _client = client; - private static readonly JsonSerializerOptions JsonSerializerOptions = - new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; - - public string OutboxId => ServalPlatformOutboxConstants.OutboxId; - - public async Task HandleMessageAsync( - string method, - string? content, - Stream? contentStream, - CancellationToken cancellationToken = default - ) - { - switch (method) - { - case ServalPlatformOutboxConstants.BuildStarted: - await _client.BuildStartedAsync( - JsonSerializer.Deserialize(content!), - cancellationToken: cancellationToken - ); - break; - case ServalPlatformOutboxConstants.BuildCompleted: - await _client.BuildCompletedAsync( - JsonSerializer.Deserialize(content!), - cancellationToken: cancellationToken - ); - break; - case ServalPlatformOutboxConstants.BuildCanceled: - await _client.BuildCanceledAsync( - JsonSerializer.Deserialize(content!), - cancellationToken: cancellationToken - ); - break; - case ServalPlatformOutboxConstants.BuildFaulted: - await _client.BuildFaultedAsync( - JsonSerializer.Deserialize(content!), - cancellationToken: cancellationToken - ); - break; - case ServalPlatformOutboxConstants.BuildRestarting: - await _client.BuildRestartingAsync( - JsonSerializer.Deserialize(content!), - cancellationToken: cancellationToken - ); - break; - case ServalPlatformOutboxConstants.InsertPretranslations: - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable( - contentStream!, - JsonSerializerOptions, - cancellationToken - ) - .OfType(); - - using (var call = _client.InsertPretranslations(cancellationToken: cancellationToken)) - { - await foreach (Pretranslation pretranslation in pretranslations) - { - await call.RequestStream.WriteAsync( - new InsertPretranslationRequest - { - EngineId = content!, - CorpusId = pretranslation.CorpusId, - TextId = pretranslation.TextId, - Refs = { pretranslation.Refs }, - Translation = pretranslation.Translation - }, - cancellationToken - ); - } - await call.RequestStream.CompleteAsync(); - await call; - } - break; - case ServalPlatformOutboxConstants.IncrementTranslationEngineCorpusSize: - await _client.IncrementTranslationEngineCorpusSizeAsync( - JsonSerializer.Deserialize(content!), - cancellationToken: cancellationToken - ); - break; - default: - throw new InvalidOperationException($"Encountered a message with the unrecognized method '{method}'."); - } - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/ServalPlatformService.cs b/src/SIL.Machine.AspNetCore/Services/ServalPlatformService.cs deleted file mode 100644 index cdd2e0d05..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ServalPlatformService.cs +++ /dev/null @@ -1,140 +0,0 @@ -using Serval.Translation.V1; - -namespace SIL.Machine.AspNetCore.Services; - -public class ServalPlatformService( - TranslationPlatformApi.TranslationPlatformApiClient client, - IMessageOutboxService outboxService -) : IPlatformService -{ - private readonly TranslationPlatformApi.TranslationPlatformApiClient _client = client; - private readonly IMessageOutboxService _outboxService = outboxService; - - public async Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default) - { - await _outboxService.EnqueueMessageAsync( - ServalPlatformOutboxConstants.OutboxId, - ServalPlatformOutboxConstants.BuildStarted, - buildId, - JsonSerializer.Serialize(new BuildStartedRequest { BuildId = buildId }), - cancellationToken: cancellationToken - ); - } - - public async Task BuildCompletedAsync( - string buildId, - int trainSize, - double confidence, - CancellationToken cancellationToken = default - ) - { - await _outboxService.EnqueueMessageAsync( - ServalPlatformOutboxConstants.OutboxId, - ServalPlatformOutboxConstants.BuildCompleted, - buildId, - JsonSerializer.Serialize( - new BuildCompletedRequest - { - BuildId = buildId, - CorpusSize = trainSize, - Confidence = confidence - } - ), - cancellationToken: cancellationToken - ); - } - - public async Task BuildCanceledAsync(string buildId, CancellationToken cancellationToken = default) - { - await _outboxService.EnqueueMessageAsync( - ServalPlatformOutboxConstants.OutboxId, - ServalPlatformOutboxConstants.BuildCanceled, - buildId, - JsonSerializer.Serialize(new BuildCanceledRequest { BuildId = buildId }), - cancellationToken: cancellationToken - ); - } - - public async Task BuildFaultedAsync(string buildId, string message, CancellationToken cancellationToken = default) - { - await _outboxService.EnqueueMessageAsync( - ServalPlatformOutboxConstants.OutboxId, - ServalPlatformOutboxConstants.BuildFaulted, - buildId, - JsonSerializer.Serialize(new BuildFaultedRequest { BuildId = buildId, Message = message }), - cancellationToken: cancellationToken - ); - } - - public async Task BuildRestartingAsync(string buildId, CancellationToken cancellationToken = default) - { - await _outboxService.EnqueueMessageAsync( - ServalPlatformOutboxConstants.OutboxId, - ServalPlatformOutboxConstants.BuildRestarting, - buildId, - JsonSerializer.Serialize(new BuildRestartingRequest { BuildId = buildId }), - cancellationToken: cancellationToken - ); - } - - public async Task UpdateBuildStatusAsync( - string buildId, - ProgressStatus progressStatus, - int? queueDepth = null, - CancellationToken cancellationToken = default - ) - { - var request = new UpdateBuildStatusRequest { BuildId = buildId, Step = progressStatus.Step }; - if (progressStatus.PercentCompleted.HasValue) - request.PercentCompleted = progressStatus.PercentCompleted.Value; - if (progressStatus.Message is not null) - request.Message = progressStatus.Message; - if (queueDepth is not null) - request.QueueDepth = queueDepth.Value; - - // just try to send it - if it fails, it fails. - await _client.UpdateBuildStatusAsync(request, cancellationToken: cancellationToken); - } - - public async Task UpdateBuildStatusAsync(string buildId, int step, CancellationToken cancellationToken = default) - { - // just try to send it - if it fails, it fails. - await _client.UpdateBuildStatusAsync( - new UpdateBuildStatusRequest { BuildId = buildId, Step = step }, - cancellationToken: cancellationToken - ); - } - - public async Task InsertPretranslationsAsync( - string engineId, - Stream pretranslationsStream, - CancellationToken cancellationToken = default - ) - { - await _outboxService.EnqueueMessageAsync( - ServalPlatformOutboxConstants.OutboxId, - ServalPlatformOutboxConstants.InsertPretranslations, - engineId, - engineId, - pretranslationsStream, - cancellationToken: cancellationToken - ); - } - - public async Task IncrementTrainSizeAsync( - string engineId, - int count = 1, - CancellationToken cancellationToken = default - ) - { - await _outboxService.EnqueueMessageAsync( - ServalPlatformOutboxConstants.OutboxId, - ServalPlatformOutboxConstants.IncrementTranslationEngineCorpusSize, - engineId, - JsonSerializer.Serialize( - new IncrementTranslationEngineCorpusSizeRequest { EngineId = engineId, Count = count } - ), - cancellationToken: cancellationToken - ); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/ServalTranslationEngineServiceV1.cs b/src/SIL.Machine.AspNetCore/Services/ServalTranslationEngineServiceV1.cs deleted file mode 100644 index 78304ac81..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ServalTranslationEngineServiceV1.cs +++ /dev/null @@ -1,341 +0,0 @@ -using Google.Protobuf.WellKnownTypes; -using Serval.Translation.V1; - -namespace SIL.Machine.AspNetCore.Services; - -public class ServalTranslationEngineServiceV1( - IEnumerable engineServices, - HealthCheckService healthCheckService -) : TranslationEngineApi.TranslationEngineApiBase -{ - private static readonly Empty Empty = new(); - - private readonly Dictionary _engineServices = - engineServices.ToDictionary(es => es.Type); - - private readonly HealthCheckService _healthCheckService = healthCheckService; - - public override async Task Create(CreateRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - TranslationEngine translationEngine = await engineService.CreateAsync( - request.EngineId, - request.HasEngineName ? request.EngineName : null, - request.SourceLanguage, - request.TargetLanguage, - request.HasIsModelPersisted ? request.IsModelPersisted : null, - context.CancellationToken - ); - return new CreateResponse { IsModelPersisted = translationEngine.IsModelPersisted }; - } - - public override async Task Delete(DeleteRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - await engineService.DeleteAsync(request.EngineId, context.CancellationToken); - return Empty; - } - - public override async Task Translate(TranslateRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - IEnumerable results; - try - { - results = await engineService.TranslateAsync( - request.EngineId, - request.N, - request.Segment, - context.CancellationToken - ); - } - catch (EngineNotBuiltException e) - { - throw new RpcException(new Status(StatusCode.Aborted, e.Message, e)); - } - - return new TranslateResponse { Results = { results.Select(Map) } }; - } - - public override async Task GetWordGraph( - GetWordGraphRequest request, - ServerCallContext context - ) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - Translation.WordGraph wordGraph; - try - { - wordGraph = await engineService.GetWordGraphAsync( - request.EngineId, - request.Segment, - context.CancellationToken - ); - } - catch (EngineNotBuiltException e) - { - throw new RpcException(new Status(StatusCode.Aborted, e.Message, e)); - } - return new GetWordGraphResponse { WordGraph = Map(wordGraph) }; - } - - public override async Task TrainSegmentPair(TrainSegmentPairRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - await engineService.TrainSegmentPairAsync( - request.EngineId, - request.SourceSegment, - request.TargetSegment, - request.SentenceStart, - context.CancellationToken - ); - return Empty; - } - - public override async Task StartBuild(StartBuildRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - Models.Corpus[] corpora = request.Corpora.Select(Map).ToArray(); - try - { - await engineService.StartBuildAsync( - request.EngineId, - request.BuildId, - request.HasOptions ? request.Options : null, - corpora, - context.CancellationToken - ); - } - catch (InvalidOperationException e) - { - throw new RpcException(new Status(StatusCode.Aborted, e.Message, e)); - } - return Empty; - } - - public override async Task CancelBuild(CancelBuildRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - try - { - await engineService.CancelBuildAsync(request.EngineId, context.CancellationToken); - } - catch (InvalidOperationException e) - { - throw new RpcException(new Status(StatusCode.Aborted, e.Message, e)); - } - return Empty; - } - - public override async Task GetModelDownloadUrl( - GetModelDownloadUrlRequest request, - ServerCallContext context - ) - { - try - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - ModelDownloadUrl modelDownloadUrl = await engineService.GetModelDownloadUrlAsync( - request.EngineId, - context.CancellationToken - ); - return new GetModelDownloadUrlResponse - { - Url = modelDownloadUrl.Url, - ModelRevision = modelDownloadUrl.ModelRevision, - ExpiresAt = modelDownloadUrl.ExpiresAt.ToTimestamp() - }; - } - catch (InvalidOperationException e) - { - throw new RpcException(new Status(StatusCode.Aborted, e.Message)); - } - catch (FileNotFoundException e) - { - throw new RpcException(new Status(StatusCode.NotFound, e.Message)); - } - } - - public override Task GetQueueSize(GetQueueSizeRequest request, ServerCallContext context) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - return Task.FromResult(new GetQueueSizeResponse { Size = engineService.GetQueueSize() }); - } - - public override Task GetLanguageInfo( - GetLanguageInfoRequest request, - ServerCallContext context - ) - { - ITranslationEngineService engineService = GetEngineService(request.EngineType); - bool isNative = engineService.IsLanguageNativeToModel(request.Language, out string internalCode); - return Task.FromResult(new GetLanguageInfoResponse { InternalCode = internalCode, IsNative = isNative, }); - } - - public override async Task HealthCheck(Empty request, ServerCallContext context) - { - HealthReport healthReport = await _healthCheckService.CheckHealthAsync(); - HealthCheckResponse healthCheckResponse = WriteGrpcHealthCheckResponse.Generate(healthReport); - return healthCheckResponse; - } - - private ITranslationEngineService GetEngineService(string engineTypeStr) - { - if (_engineServices.TryGetValue(GetEngineType(engineTypeStr), out ITranslationEngineService? service)) - return service; - throw new RpcException(new Status(StatusCode.InvalidArgument, "The engine type is invalid.")); - } - - private static TranslationEngineType GetEngineType(string engineTypeStr) - { - engineTypeStr = engineTypeStr[0].ToString().ToUpperInvariant() + engineTypeStr[1..]; - if (System.Enum.TryParse(engineTypeStr, out TranslationEngineType engineType)) - return engineType; - throw new RpcException(new Status(StatusCode.InvalidArgument, "The engine type is invalid.")); - } - - private static Serval.Translation.V1.TranslationResult Map(Translation.TranslationResult source) - { - return new Serval.Translation.V1.TranslationResult - { - Translation = source.Translation, - SourceTokens = { source.SourceTokens }, - TargetTokens = { source.TargetTokens }, - Confidences = { source.Confidences }, - Sources = { source.Sources.Select(Map) }, - Alignment = { Map(source.Alignment) }, - Phrases = { source.Phrases.Select(Map) } - }; - } - - private static Serval.Translation.V1.WordGraph Map(Translation.WordGraph source) - { - return new Serval.Translation.V1.WordGraph - { - SourceTokens = { source.SourceTokens }, - InitialStateScore = source.InitialStateScore, - FinalStates = { source.FinalStates }, - Arcs = { source.Arcs.Select(Map) } - }; - } - - private static Serval.Translation.V1.WordGraphArc Map(Translation.WordGraphArc source) - { - return new Serval.Translation.V1.WordGraphArc - { - PrevState = source.PrevState, - NextState = source.NextState, - Score = source.Score, - TargetTokens = { source.TargetTokens }, - Alignment = { Map(source.Alignment) }, - Confidences = { source.Confidences }, - SourceSegmentStart = source.SourceSegmentRange.Start, - SourceSegmentEnd = source.SourceSegmentRange.End, - Sources = { source.Sources.Select(Map) } - }; - } - - private static Serval.Translation.V1.TranslationSources Map(Translation.TranslationSources source) - { - return new Serval.Translation.V1.TranslationSources - { - Values = - { - System - .Enum.GetValues() - .Where(s => s != Translation.TranslationSources.None && source.HasFlag(s)) - .Select(s => - s switch - { - Translation.TranslationSources.Smt => TranslationSource.Primary, - Translation.TranslationSources.Nmt => TranslationSource.Primary, - Translation.TranslationSources.Transfer => TranslationSource.Secondary, - Translation.TranslationSources.Prefix => TranslationSource.Human, - _ => TranslationSource.Primary - } - ) - } - }; - } - - private static IEnumerable Map(WordAlignmentMatrix source) - { - for (int i = 0; i < source.RowCount; i++) - { - for (int j = 0; j < source.ColumnCount; j++) - { - if (source[i, j]) - yield return new Serval.Translation.V1.AlignedWordPair { SourceIndex = i, TargetIndex = j }; - } - } - } - - private static Serval.Translation.V1.Phrase Map(Translation.Phrase source) - { - return new Serval.Translation.V1.Phrase - { - SourceSegmentStart = source.SourceSegmentRange.Start, - SourceSegmentEnd = source.SourceSegmentRange.End, - TargetSegmentCut = source.TargetSegmentCut - }; - } - - private static Models.Corpus Map(Serval.Translation.V1.Corpus source) - { - var pretranslateChapters = source.PretranslateChapters.ToDictionary( - kvp => kvp.Key, - kvp => kvp.Value.Chapters.ToHashSet() - ); - FilterChoice pretranslateFilter = GetFilterChoice(source.PretranslateAll, pretranslateChapters); - - var trainOnChapters = source.TrainOnChapters.ToDictionary( - kvp => kvp.Key, - kvp => kvp.Value.Chapters.ToHashSet() - ); - FilterChoice trainingFilter = GetFilterChoice(source.TrainOnAll, trainOnChapters); - - return new Models.Corpus - { - Id = source.Id, - SourceLanguage = source.SourceLanguage, - TargetLanguage = source.TargetLanguage, - TrainOnChapters = trainingFilter == FilterChoice.Chapters ? trainOnChapters : null, - PretranslateChapters = pretranslateFilter == FilterChoice.Chapters ? pretranslateChapters : null, - TrainOnTextIds = trainingFilter == FilterChoice.TextIds ? source.TrainOnTextIds.ToHashSet() : null, - PretranslateTextIds = - pretranslateFilter == FilterChoice.TextIds ? source.PretranslateTextIds.ToHashSet() : null, - SourceFiles = source.SourceFiles.Select(Map).ToList(), - TargetFiles = source.TargetFiles.Select(Map).ToList() - }; - } - - private static Models.CorpusFile Map(Serval.Translation.V1.CorpusFile source) - { - return new Models.CorpusFile - { - Location = source.Location, - Format = (Models.FileFormat)source.Format, - TextId = source.TextId - }; - } - - private enum FilterChoice - { - Chapters, - TextIds, - None - } - - private static FilterChoice GetFilterChoice(bool all, IReadOnlyDictionary> chapters) - { - if (all) - return FilterChoice.None; - - // Only either textIds or Scripture Range will be used at a time - // TextIds may be an empty array, so prefer that if both are empty (which applies to both scripture and text) - if (chapters.Count == 0) - return FilterChoice.TextIds; - else - return FilterChoice.Chapters; - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/SharedFileService.cs b/src/SIL.Machine.AspNetCore/Services/SharedFileService.cs deleted file mode 100644 index b4244211e..000000000 --- a/src/SIL.Machine.AspNetCore/Services/SharedFileService.cs +++ /dev/null @@ -1,104 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SharedFileService : ISharedFileService -{ - private readonly Uri? _baseUri; - private readonly IFileStorage _fileStorage; - private readonly bool _supportFolderDelete = true; - private readonly ILoggerFactory _loggerFactory; - - public SharedFileService(ILoggerFactory loggerFactory, IOptions? options = null) - { - _loggerFactory = loggerFactory; - - if (options?.Value.Uri is null) - { - _fileStorage = new InMemoryStorage(); - } - else - { - string baseUri = options.Value.Uri; - if (!baseUri.EndsWith("/")) - baseUri += "/"; - _baseUri = new Uri(baseUri); - switch (_baseUri.Scheme) - { - case "file": - _fileStorage = new LocalStorage(_baseUri.LocalPath); - break; - case "s3": - _fileStorage = new S3FileStorage( - _baseUri.Host, - _baseUri.AbsolutePath, - options.Value.S3AccessKeyId, - options.Value.S3SecretAccessKey, - options.Value.S3Region, - _loggerFactory - ); - _supportFolderDelete = false; - break; - default: - throw new InvalidOperationException($"Unsupported URI scheme: {_baseUri.Scheme}"); - } - } - } - - public Uri GetBaseUri() - { - return GetResolvedUri(""); - } - - public Uri GetResolvedUri(string path) - { - if (_baseUri is null) - return new Uri($"memory://{path}"); - return new Uri(_baseUri, path); - } - - public async Task GetDownloadUrlAsync(string path, DateTime expiresAt) - { - return await _fileStorage.GetDownloadUrlAsync(path, expiresAt); - } - - public Task> ListFilesAsync( - string path, - bool recurse = false, - CancellationToken cancellationToken = default - ) - { - return _fileStorage.ListFilesAsync(path, recurse, cancellationToken); - } - - public Task OpenReadAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenReadAsync(path, cancellationToken); - } - - public Task OpenWriteAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.OpenWriteAsync(path, cancellationToken); - } - - public async Task DeleteAsync(string path, CancellationToken cancellationToken = default) - { - if (!_supportFolderDelete && path.EndsWith("/")) - { - IReadOnlyCollection files = await _fileStorage.ListFilesAsync( - path, - recurse: true, - cancellationToken - ); - foreach (string file in files) - await _fileStorage.DeleteAsync(file, cancellationToken: cancellationToken); - } - else - { - await _fileStorage.DeleteAsync(path, recurse: true, cancellationToken: cancellationToken); - } - } - - public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - { - return _fileStorage.ExistsAsync(path, cancellationToken); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferBuildJob.cs deleted file mode 100644 index 21f16b324..000000000 --- a/src/SIL.Machine.AspNetCore/Services/SmtTransferBuildJob.cs +++ /dev/null @@ -1,157 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, - ILogger logger, - IRepository trainSegmentPairs, - ITruecaserFactory truecaserFactory, - ISmtModelFactory smtModelFactory, - ICorpusService corpusService -) - : HangfireBuildJob>( - platformService, - engines, - lockFactory, - dataAccessContext, - buildJobService, - logger - ) -{ - private readonly IRepository _trainSegmentPairs = trainSegmentPairs; - private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; - private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; - private readonly ICorpusService _corpusService = corpusService; - - protected override Task InitializeAsync( - string engineId, - string buildId, - IReadOnlyList data, - IDistributedReaderWriterLock @lock, - CancellationToken cancellationToken - ) - { - return _trainSegmentPairs.DeleteAllAsync(p => p.TranslationEngineRef == engineId, cancellationToken); - } - - protected override async Task DoWorkAsync( - string engineId, - string buildId, - IReadOnlyList data, - string? buildOptions, - IDistributedReaderWriterLock @lock, - CancellationToken cancellationToken - ) - { - await PlatformService.BuildStartedAsync(buildId, cancellationToken); - Logger.LogInformation("Build started ({0})", buildId); - var stopwatch = new Stopwatch(); - stopwatch.Start(); - - cancellationToken.ThrowIfCancellationRequested(); - - JsonObject? buildOptionsObject = null; - if (buildOptions is not null) - { - buildOptionsObject = JsonSerializer.Deserialize(buildOptions); - } - - var targetCorpora = new List(); - var parallelCorpora = new List(); - foreach (Corpus corpus in data) - { - ITextCorpus? sourceTextCorpus = _corpusService.CreateTextCorpora(corpus.SourceFiles).FirstOrDefault(); - ITextCorpus? targetTextCorpus = _corpusService.CreateTextCorpora(corpus.TargetFiles).FirstOrDefault(); - if (sourceTextCorpus is null || targetTextCorpus is null) - continue; - - targetCorpora.Add(targetTextCorpus); - parallelCorpora.Add(sourceTextCorpus.AlignRows(targetTextCorpus)); - - if ((bool?)buildOptionsObject?["use_key_terms"] ?? true) - { - ITextCorpus? sourceTermCorpus = _corpusService.CreateTermCorpora(corpus.SourceFiles).FirstOrDefault(); - ITextCorpus? targetTermCorpus = _corpusService.CreateTermCorpora(corpus.TargetFiles).FirstOrDefault(); - if (sourceTermCorpus is not null && targetTermCorpus is not null) - { - IParallelTextCorpus parallelKeyTermsCorpus = sourceTermCorpus.AlignRows(targetTermCorpus); - parallelCorpora.Add(parallelKeyTermsCorpus); - } - } - } - - IParallelTextCorpus parallelCorpus = parallelCorpora.Flatten(); - ITextCorpus targetCorpus = targetCorpora.Flatten(); - - var tokenizer = new LatinWordTokenizer(); - var detokenizer = new LatinWordDetokenizer(); - - using ITrainer smtModelTrainer = await _smtModelFactory.CreateTrainerAsync(engineId, tokenizer, parallelCorpus); - using ITrainer truecaseTrainer = await _truecaserFactory.CreateTrainerAsync(engineId, tokenizer, targetCorpus); - - cancellationToken.ThrowIfCancellationRequested(); - - var progress = new BuildProgress(PlatformService, buildId); - await smtModelTrainer.TrainAsync(progress, cancellationToken); - await truecaseTrainer.TrainAsync(cancellationToken: cancellationToken); - - TranslationEngine? engine = await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new OperationCanceledException(); - - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - cancellationToken.ThrowIfCancellationRequested(); - await smtModelTrainer.SaveAsync(CancellationToken.None); - await truecaseTrainer.SaveAsync(CancellationToken.None); - ITruecaser truecaser = await _truecaserFactory.CreateAsync(engineId); - IReadOnlyList segmentPairs = await _trainSegmentPairs.GetAllAsync( - p => p.TranslationEngineRef == engine.Id, - CancellationToken.None - ); - using ( - IInteractiveTranslationModel smtModel = await _smtModelFactory.CreateAsync( - engineId, - tokenizer, - detokenizer, - truecaser - ) - ) - { - foreach (TrainSegmentPair segmentPair in segmentPairs) - { - await smtModel.TrainSegmentAsync( - segmentPair.Source, - segmentPair.Target, - cancellationToken: CancellationToken.None - ); - } - } - - await DataAccessContext.WithTransactionAsync( - async (ct) => - { - await PlatformService.BuildCompletedAsync( - buildId, - smtModelTrainer.Stats.TrainCorpusSize + segmentPairs.Count, - smtModelTrainer.Stats.Metrics["bleu"] * 100.0, - CancellationToken.None - ); - await BuildJobService.BuildJobFinishedAsync( - engineId, - buildId, - buildComplete: true, - CancellationToken.None - ); - }, - cancellationToken: CancellationToken.None - ); - } - - stopwatch.Stop(); - Logger.LogInformation("Build completed in {0}s ({1})", stopwatch.Elapsed.TotalSeconds, buildId); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferClearMLBuildJobFactory.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferClearMLBuildJobFactory.cs deleted file mode 100644 index 6a1f42aeb..000000000 --- a/src/SIL.Machine.AspNetCore/Services/SmtTransferClearMLBuildJobFactory.cs +++ /dev/null @@ -1,49 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferClearMLBuildJobFactory( - ISharedFileService sharedFileService, - IRepository engines -) : IClearMLBuildJobFactory -{ - private readonly ISharedFileService _sharedFileService = sharedFileService; - private readonly IRepository _engines = engines; - - public TranslationEngineType EngineType => TranslationEngineType.SmtTransfer; - - public async Task CreateJobScriptAsync( - string engineId, - string buildId, - string modelType, - BuildStage stage, - object? data = null, - string? buildOptions = null, - CancellationToken cancellationToken = default - ) - { - if (stage == BuildStage.Train) - { - TranslationEngine? engine = await _engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException("The engine does not exist."); - - Uri sharedFileUri = _sharedFileService.GetBaseUri(); - string baseUri = sharedFileUri.GetComponents(UriComponents.SchemeAndServer, UriFormat.Unescaped); - string folder = sharedFileUri.GetComponents(UriComponents.Path, UriFormat.Unescaped); - return "from machine.jobs.build_smt_engine import run\n" - + "args = {\n" - + $" 'model_type': '{modelType}',\n" - + $" 'engine_id': '{engineId}',\n" - + $" 'build_id': '{buildId}',\n" - + $" 'shared_file_uri': '{baseUri}',\n" - + $" 'shared_file_folder': '{folder}',\n" - + (buildOptions is not null ? $" 'build_options': '''{buildOptions}''',\n" : "") - + $" 'clearml': True\n" - + "}\n" - + "run(args)\n"; - } - else - { - throw new ArgumentException("Unknown build stage.", nameof(stage)); - } - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineCommitService.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineCommitService.cs deleted file mode 100644 index 91fdc5f2f..000000000 --- a/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineCommitService.cs +++ /dev/null @@ -1,38 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineCommitService( - IServiceProvider services, - IOptionsMonitor engineOptions, - SmtTransferEngineStateService stateService, - ILogger logger -) - : RecurrentTask( - "SMT transfer engine commit service", - services, - engineOptions.CurrentValue.EngineCommitFrequency, - logger - ) -{ - private readonly IOptionsMonitor _engineOptions = engineOptions; - private readonly SmtTransferEngineStateService _stateService = stateService; - private readonly ILogger _logger = logger; - - protected override async Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken) - { - try - { - var engines = scope.ServiceProvider.GetRequiredService>(); - var lockFactory = scope.ServiceProvider.GetRequiredService(); - await _stateService.CommitAsync( - lockFactory, - engines, - _engineOptions.CurrentValue.InactiveEngineTimeout, - cancellationToken - ); - } - catch (Exception e) - { - _logger.LogError(e, "Error occurred while committing SMT transfer engines."); - } - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService.cs deleted file mode 100644 index c00910487..000000000 --- a/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineService.cs +++ /dev/null @@ -1,272 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineService( - IDistributedReaderWriterLockFactory lockFactory, - IPlatformService platformService, - IDataAccessContext dataAccessContext, - IRepository engines, - IRepository trainSegmentPairs, - SmtTransferEngineStateService stateService, - IBuildJobService buildJobService, - IClearMLQueueService clearMLQueueService -) : ITranslationEngineService -{ - private readonly IDistributedReaderWriterLockFactory _lockFactory = lockFactory; - private readonly IPlatformService _platformService = platformService; - private readonly IDataAccessContext _dataAccessContext = dataAccessContext; - private readonly IRepository _engines = engines; - private readonly IRepository _trainSegmentPairs = trainSegmentPairs; - private readonly SmtTransferEngineStateService _stateService = stateService; - private readonly IBuildJobService _buildJobService = buildJobService; - private readonly IClearMLQueueService _clearMLQueueService = clearMLQueueService; - - public TranslationEngineType Type => TranslationEngineType.SmtTransfer; - - public async Task CreateAsync( - string engineId, - string? engineName, - string sourceLanguage, - string targetLanguage, - bool? isModelPersisted = null, - CancellationToken cancellationToken = default - ) - { - if (isModelPersisted == false) - { - throw new NotSupportedException( - "SMT transfer engines do not support non-persisted models." - + "Please remove the isModelPersisted parameter or set it to true." - ); - } - - TranslationEngine translationEngine = await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - var translationEngine = new TranslationEngine - { - EngineId = engineId, - SourceLanguage = sourceLanguage, - TargetLanguage = targetLanguage, - Type = TranslationEngineType.SmtTransfer, - IsModelPersisted = isModelPersisted ?? true // models are persisted if not specified - }; - await _engines.InsertAsync(translationEngine, ct); - await _buildJobService.CreateEngineAsync(engineId, engineName, ct); - return translationEngine; - }, - cancellationToken: cancellationToken - ); - - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, CancellationToken.None); - await using (await @lock.WriterLockAsync(cancellationToken: CancellationToken.None)) - { - SmtTransferEngineState state = _stateService.Get(engineId); - await state.InitNewAsync(CancellationToken.None); - } - return translationEngine; - } - - public async Task DeleteAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - await CancelBuildJobAsync(engineId, cancellationToken); - - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - await _engines.DeleteAsync(e => e.EngineId == engineId, ct); - await _trainSegmentPairs.DeleteAllAsync(p => p.TranslationEngineRef == engineId, ct); - }, - cancellationToken: cancellationToken - ); - await _buildJobService.DeleteEngineAsync(engineId, CancellationToken.None); - - if (_stateService.TryRemove(engineId, out SmtTransferEngineState? state)) - { - await state.DeleteDataAsync(); - await state.DisposeAsync(); - } - } - await _lockFactory.DeleteAsync(engineId, CancellationToken.None); - } - - public async Task> TranslateAsync( - string engineId, - int n, - string segment, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - SmtTransferEngineState state = _stateService.Get(engineId); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - IReadOnlyList results = await hybridEngine.TranslateAsync(n, segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return results; - } - } - - public async Task GetWordGraphAsync( - string engineId, - string segment, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.ReaderLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetBuiltEngineAsync(engineId, cancellationToken); - SmtTransferEngineState state = _stateService.Get(engineId); - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - WordGraph result = await hybridEngine.GetWordGraphAsync(segment, cancellationToken); - state.LastUsedTime = DateTime.Now; - return result; - } - } - - public async Task TrainSegmentPairAsync( - string engineId, - string sourceSegment, - string targetSegment, - bool sentenceStart, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - - async Task TrainSubroutineAsync(SmtTransferEngineState state, CancellationToken ct) - { - HybridTranslationEngine hybridEngine = await state.GetHybridEngineAsync(engine.BuildRevision); - await hybridEngine.TrainSegmentAsync(sourceSegment, targetSegment, sentenceStart, ct); - await _platformService.IncrementTrainSizeAsync(engineId, cancellationToken: CancellationToken.None); - } - - SmtTransferEngineState state = _stateService.Get(engineId); - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - if (engine.CurrentBuild?.JobState is BuildJobState.Active) - { - await _trainSegmentPairs.InsertAsync( - new TrainSegmentPair - { - TranslationEngineRef = engineId, - Source = sourceSegment, - Target = targetSegment, - SentenceStart = sentenceStart - }, - CancellationToken.None - ); - await TrainSubroutineAsync(state, CancellationToken.None); - } - else - { - await TrainSubroutineAsync(state, ct); - } - }, - cancellationToken: cancellationToken - ); - - state.IsUpdated = true; - state.LastUsedTime = DateTime.Now; - } - } - - public async Task StartBuildAsync( - string engineId, - string buildId, - string? buildOptions, - IReadOnlyList corpora, - CancellationToken cancellationToken = default - ) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - // If there is a pending/running build, then no need to start a new one. - if (await _buildJobService.IsEngineBuilding(engineId, cancellationToken)) - throw new InvalidOperationException("The engine is already building or in the process of canceling."); - - await _buildJobService.StartBuildJobAsync( - BuildJobRunnerType.Hangfire, - engineId, - buildId, - BuildStage.Preprocess, - corpora, - buildOptions, - cancellationToken - ); - SmtTransferEngineState state = _stateService.Get(engineId); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public async Task CancelBuildAsync(string engineId, CancellationToken cancellationToken = default) - { - IDistributedReaderWriterLock @lock = await _lockFactory.CreateAsync(engineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - if (!await CancelBuildJobAsync(engineId, cancellationToken)) - throw new InvalidOperationException("The engine is not currently building."); - SmtTransferEngineState state = _stateService.Get(engineId); - state.LastUsedTime = DateTime.UtcNow; - } - } - - public int GetQueueSize() - { - return _clearMLQueueService.GetQueueSize(Type); - } - - public bool IsLanguageNativeToModel(string language, out string internalCode) - { - throw new NotSupportedException("SMT transfer engines do not support language info."); - } - - private async Task CancelBuildJobAsync(string engineId, CancellationToken cancellationToken) - { - string? buildId = null; - await _dataAccessContext.WithTransactionAsync( - async (ct) => - { - (buildId, BuildJobState jobState) = await _buildJobService.CancelBuildJobAsync(engineId, ct); - if (buildId is not null && jobState is BuildJobState.None) - await _platformService.BuildCanceledAsync(buildId, CancellationToken.None); - }, - cancellationToken: cancellationToken - ); - return buildId is not null; - } - - public Task GetModelDownloadUrlAsync( - string engineId, - CancellationToken cancellationToken = default - ) - { - throw new NotSupportedException(); - } - - private async Task GetEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine? engine = await _engines.GetAsync(e => e.EngineId == engineId, cancellationToken); - if (engine is null) - throw new InvalidOperationException($"The engine {engineId} does not exist."); - return engine; - } - - private async Task GetBuiltEngineAsync(string engineId, CancellationToken cancellationToken) - { - TranslationEngine engine = await GetEngineAsync(engineId, cancellationToken); - if (engine.BuildRevision == 0) - throw new EngineNotBuiltException("The engine must be built first."); - return engine; - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineState.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineState.cs deleted file mode 100644 index d76350989..000000000 --- a/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineState.cs +++ /dev/null @@ -1,128 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineState( - ISmtModelFactory smtModelFactory, - ITransferEngineFactory transferEngineFactory, - ITruecaserFactory truecaserFactory, - IOptionsMonitor options, - string engineId -) : AsyncDisposableBase -{ - private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; - private readonly ITransferEngineFactory _transferEngineFactory = transferEngineFactory; - private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; - private readonly IOptionsMonitor _options = options; - private readonly AsyncLock _lock = new(); - - private IInteractiveTranslationModel? _smtModel; - private HybridTranslationEngine? _hybridEngine; - - public string EngineId { get; } = engineId; - - public bool IsUpdated { get; set; } - public int CurrentBuildRevision { get; set; } = -1; - public DateTime LastUsedTime { get; set; } = DateTime.UtcNow; - public bool IsLoaded => _hybridEngine != null; - - private string EngineDir => Path.Combine(_options.CurrentValue.EnginesDir, EngineId); - - public async Task InitNewAsync(CancellationToken cancellationToken = default) - { - await _smtModelFactory.InitNewAsync(EngineDir, cancellationToken); - await _transferEngineFactory.InitNewAsync(EngineDir, cancellationToken); - } - - public async Task GetHybridEngineAsync(int buildRevision) - { - using (await _lock.LockAsync()) - { - if (_hybridEngine is not null && CurrentBuildRevision != -1 && buildRevision != CurrentBuildRevision) - { - IsUpdated = false; - await UnloadAsync(); - } - - if (_hybridEngine is null) - { - LatinWordTokenizer tokenizer = new(); - LatinWordDetokenizer detokenizer = new(); - ITruecaser truecaser = await _truecaserFactory.CreateAsync(EngineDir); - _smtModel = await _smtModelFactory.CreateAsync(EngineDir, tokenizer, detokenizer, truecaser); - ITranslationEngine? transferEngine = await _transferEngineFactory.CreateAsync( - EngineDir, - tokenizer, - detokenizer, - truecaser - ); - _hybridEngine = new HybridTranslationEngine(_smtModel, transferEngine) - { - TargetDetokenizer = detokenizer - }; - } - CurrentBuildRevision = buildRevision; - return _hybridEngine; - } - } - - public async Task DeleteDataAsync() - { - await UnloadAsync(); - await _smtModelFactory.CleanupAsync(EngineDir); - await _transferEngineFactory.CleanupAsync(EngineDir); - await _truecaserFactory.CleanupAsync(EngineDir); - } - - public async Task CommitAsync( - int buildRevision, - TimeSpan inactiveTimeout, - CancellationToken cancellationToken = default - ) - { - if (_hybridEngine is null) - return; - - if (CurrentBuildRevision == -1) - CurrentBuildRevision = buildRevision; - if (buildRevision != CurrentBuildRevision) - { - await UnloadAsync(cancellationToken); - CurrentBuildRevision = buildRevision; - } - else if (DateTime.Now - LastUsedTime > inactiveTimeout) - { - await UnloadAsync(cancellationToken); - } - else - { - await SaveModelAsync(cancellationToken); - } - } - - private async Task SaveModelAsync(CancellationToken cancellationToken = default) - { - if (_smtModel is not null && IsUpdated) - { - await _smtModel.SaveAsync(cancellationToken); - IsUpdated = false; - } - } - - private async Task UnloadAsync(CancellationToken cancellationToken = default) - { - if (_hybridEngine is null) - return; - - await SaveModelAsync(cancellationToken); - - _hybridEngine.Dispose(); - - _smtModel = null; - _hybridEngine = null; - CurrentBuildRevision = -1; - } - - protected override async ValueTask DisposeAsyncCore() - { - await UnloadAsync(); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineStateService.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineStateService.cs deleted file mode 100644 index 66f4ebcf2..000000000 --- a/src/SIL.Machine.AspNetCore/Services/SmtTransferEngineStateService.cs +++ /dev/null @@ -1,72 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferEngineStateService( - ISmtModelFactory smtModelFactory, - ITransferEngineFactory transferEngineFactory, - ITruecaserFactory truecaserFactory, - IOptionsMonitor options -) : AsyncDisposableBase -{ - private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; - private readonly ITransferEngineFactory _transferEngineFactory = transferEngineFactory; - private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; - private readonly IOptionsMonitor _options = options; - - private readonly ConcurrentDictionary _engineStates = - new ConcurrentDictionary(); - - public SmtTransferEngineState Get(string engineId) - { - return _engineStates.GetOrAdd(engineId, CreateState); - } - - public bool TryRemove(string engineId, [MaybeNullWhen(false)] out SmtTransferEngineState state) - { - return _engineStates.TryRemove(engineId, out state); - } - - public async Task CommitAsync( - IDistributedReaderWriterLockFactory lockFactory, - IRepository engines, - TimeSpan inactiveTimeout, - CancellationToken cancellationToken = default - ) - { - foreach (SmtTransferEngineState state in _engineStates.Values) - { - IDistributedReaderWriterLock @lock = await lockFactory.CreateAsync(state.EngineId, cancellationToken); - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - TranslationEngine? engine = await engines.GetAsync( - e => e.EngineId == state.EngineId, - cancellationToken - ); - if ( - engine is not null - && (engine.CurrentBuild is null || engine.CurrentBuild.JobState is BuildJobState.Pending) - ) - { - await state.CommitAsync(engine.BuildRevision, inactiveTimeout, cancellationToken); - } - } - } - } - - private SmtTransferEngineState CreateState(string engineId) - { - return new SmtTransferEngineState( - _smtModelFactory, - _transferEngineFactory, - _truecaserFactory, - _options, - engineId - ); - } - - protected override async ValueTask DisposeAsyncCore() - { - foreach (SmtTransferEngineState state in _engineStates.Values) - await state.DisposeAsync(); - _engineStates.Clear(); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferHangfireBuildJobFactory.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferHangfireBuildJobFactory.cs deleted file mode 100644 index efbeb1207..000000000 --- a/src/SIL.Machine.AspNetCore/Services/SmtTransferHangfireBuildJobFactory.cs +++ /dev/null @@ -1,33 +0,0 @@ -using static SIL.Machine.AspNetCore.Services.HangfireBuildJobRunner; - -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferHangfireBuildJobFactory : IHangfireBuildJobFactory -{ - public TranslationEngineType EngineType => TranslationEngineType.SmtTransfer; - - public Job CreateJob(string engineId, string buildId, BuildStage stage, object? data, string? buildOptions) - { - return stage switch - { - BuildStage.Preprocess - => CreateJob>( - engineId, - buildId, - "smt_transfer", - data, - buildOptions - ), - BuildStage.Postprocess - => CreateJob( - engineId, - buildId, - "smt_transfer", - data, - buildOptions - ), - BuildStage.Train => CreateJob(engineId, buildId, "smt_transfer", buildOptions), - _ => throw new ArgumentException("Unknown build stage.", nameof(stage)), - }; - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferPostprocessBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferPostprocessBuildJob.cs deleted file mode 100644 index f2b6b438b..000000000 --- a/src/SIL.Machine.AspNetCore/Services/SmtTransferPostprocessBuildJob.cs +++ /dev/null @@ -1,74 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferPostprocessBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, - ILogger logger, - ISharedFileService sharedFileService, - IRepository trainSegmentPairs, - ISmtModelFactory smtModelFactory, - ITruecaserFactory truecaserFactory, - IOptionsMonitor options -) - : PostprocessBuildJob( - platformService, - engines, - lockFactory, - dataAccessContext, - buildJobService, - logger, - sharedFileService - ) -{ - private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; - private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; - private readonly IRepository _trainSegmentPairs = trainSegmentPairs; - private readonly IOptionsMonitor _options = options; - - protected override async Task SaveModelAsync(string engineId, string buildId) - { - await using ( - Stream engineStream = await SharedFileService.OpenReadAsync( - $"builds/{buildId}/model.tar.gz", - CancellationToken.None - ) - ) - { - await _smtModelFactory.UpdateEngineFromAsync( - Path.Combine(_options.CurrentValue.EnginesDir, engineId), - engineStream, - CancellationToken.None - ); - } - return await TrainOnNewSegmentPairsAsync(engineId); - } - - private async Task TrainOnNewSegmentPairsAsync(string engineId) - { - IReadOnlyList segmentPairs = await _trainSegmentPairs.GetAllAsync(p => - p.TranslationEngineRef == engineId - ); - if (segmentPairs.Count == 0) - return segmentPairs.Count; - - string engineDir = Path.Combine(_options.CurrentValue.EnginesDir, engineId); - var tokenizer = new LatinWordTokenizer(); - var detokenizer = new LatinWordDetokenizer(); - ITruecaser truecaser = await _truecaserFactory.CreateAsync(engineDir); - using IInteractiveTranslationModel smtModel = await _smtModelFactory.CreateAsync( - engineDir, - tokenizer, - detokenizer, - truecaser - ); - foreach (TrainSegmentPair segmentPair in segmentPairs) - { - await smtModel.TrainSegmentAsync(segmentPair.Source, segmentPair.Target); - } - await smtModel.SaveAsync(); - return segmentPairs.Count; - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/SmtTransferTrainBuildJob.cs b/src/SIL.Machine.AspNetCore/Services/SmtTransferTrainBuildJob.cs deleted file mode 100644 index 8f2640c3b..000000000 --- a/src/SIL.Machine.AspNetCore/Services/SmtTransferTrainBuildJob.cs +++ /dev/null @@ -1,237 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class SmtTransferTrainBuildJob( - IPlatformService platformService, - IRepository engines, - IDistributedReaderWriterLockFactory lockFactory, - IDataAccessContext dataAccessContext, - IBuildJobService buildJobService, - ILogger logger, - ISharedFileService sharedFileService, - ITruecaserFactory truecaserFactory, - ISmtModelFactory smtModelFactory, - ITransferEngineFactory transferEngineFactory -) : HangfireBuildJob(platformService, engines, lockFactory, dataAccessContext, buildJobService, logger) -{ - private static readonly JsonWriterOptions PretranslateWriterOptions = new() { Indented = true }; - private static readonly JsonSerializerOptions JsonSerializerOptions = - new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; - private const int BatchSize = 128; - - private readonly ISharedFileService _sharedFileService = sharedFileService; - private readonly ITruecaserFactory _truecaserFactory = truecaserFactory; - private readonly ISmtModelFactory _smtModelFactory = smtModelFactory; - private readonly ITransferEngineFactory _transferEngineFactory = transferEngineFactory; - - protected override async Task DoWorkAsync( - string engineId, - string buildId, - object? data, - string? buildOptions, - IDistributedReaderWriterLock @lock, - CancellationToken cancellationToken - ) - { - using TempDirectory tempDir = new(buildId); - string corpusDir = Path.Combine(tempDir.Path, "corpus"); - await DownloadDataAsync(buildId, corpusDir, cancellationToken); - - // assemble corpus - ITextCorpus sourceCorpus = new TextFileTextCorpus(Path.Combine(corpusDir, "train.src.txt")); - ITextCorpus targetCorpus = new TextFileTextCorpus(Path.Combine(corpusDir, "train.trg.txt")); - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - - // train SMT model - string engineDir = Path.Combine(tempDir.Path, "engine"); - (int trainCorpusSize, double confidence) = await TrainAsync( - buildId, - engineDir, - targetCorpus, - parallelCorpus, - cancellationToken - ); - - cancellationToken.ThrowIfCancellationRequested(); - - await GeneratePretranslationsAsync(buildId, engineDir, cancellationToken); - - await using (await @lock.WriterLockAsync(cancellationToken: cancellationToken)) - { - bool canceling = !await BuildJobService.StartBuildJobAsync( - BuildJobRunnerType.Hangfire, - engineId, - buildId, - BuildStage.Postprocess, - data: (trainCorpusSize, confidence), - buildOptions: buildOptions, - cancellationToken: cancellationToken - ); - if (canceling) - throw new OperationCanceledException(); - } - } - - protected override async Task CleanupAsync( - string engineId, - string buildId, - object? data, - IDistributedReaderWriterLock @lock, - JobCompletionStatus completionStatus - ) - { - if (completionStatus is JobCompletionStatus.Canceled) - { - try - { - await _sharedFileService.DeleteAsync($"builds/{buildId}/"); - } - catch (Exception e) - { - Logger.LogWarning(e, "Unable to to delete job data for build {BuildId}.", buildId); - } - } - } - - private async Task DownloadDataAsync(string buildId, string corpusDir, CancellationToken cancellationToken) - { - Directory.CreateDirectory(corpusDir); - await using Stream srcText = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/train.src.txt", - cancellationToken - ); - await using FileStream srcFileStream = File.Create(Path.Combine(corpusDir, "train.src.txt")); - await srcText.CopyToAsync(srcFileStream, cancellationToken); - - await using Stream tgtText = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/train.trg.txt", - cancellationToken - ); - await using FileStream tgtFileStream = File.Create(Path.Combine(corpusDir, "train.trg.txt")); - await tgtText.CopyToAsync(tgtFileStream, cancellationToken); - } - - private async Task<(int TrainCorpusSize, double Confidence)> TrainAsync( - string buildId, - string engineDir, - ITextCorpus targetCorpus, - IParallelTextCorpus parallelCorpus, - CancellationToken cancellationToken - ) - { - await _smtModelFactory.InitNewAsync(engineDir, cancellationToken); - LatinWordTokenizer tokenizer = new(); - int trainCorpusSize; - double confidence; - using ITrainer smtModelTrainer = await _smtModelFactory.CreateTrainerAsync( - engineDir, - tokenizer, - parallelCorpus, - cancellationToken - ); - using ITrainer truecaseTrainer = await _truecaserFactory.CreateTrainerAsync( - engineDir, - tokenizer, - targetCorpus, - cancellationToken - ); - cancellationToken.ThrowIfCancellationRequested(); - - var progress = new BuildProgress(PlatformService, buildId); - await smtModelTrainer.TrainAsync(progress, cancellationToken); - await truecaseTrainer.TrainAsync(cancellationToken: cancellationToken); - - trainCorpusSize = smtModelTrainer.Stats.TrainCorpusSize; - confidence = smtModelTrainer.Stats.Metrics["bleu"] * 100.0; - - cancellationToken.ThrowIfCancellationRequested(); - - await smtModelTrainer.SaveAsync(cancellationToken); - await truecaseTrainer.SaveAsync(cancellationToken); - - await using Stream engineStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/model.tar.gz", - cancellationToken - ); - await _smtModelFactory.SaveEngineToAsync(engineDir, engineStream, cancellationToken); - return (trainCorpusSize, confidence); - } - - private async Task GeneratePretranslationsAsync( - string buildId, - string engineDir, - CancellationToken cancellationToken - ) - { - await using Stream sourceStream = await _sharedFileService.OpenReadAsync( - $"builds/{buildId}/pretranslate.src.json", - cancellationToken - ); - - IAsyncEnumerable pretranslations = JsonSerializer - .DeserializeAsyncEnumerable(sourceStream, JsonSerializerOptions, cancellationToken) - .OfType(); - - await using Stream targetStream = await _sharedFileService.OpenWriteAsync( - $"builds/{buildId}/pretranslate.trg.json", - cancellationToken - ); - await using Utf8JsonWriter targetWriter = new(targetStream, PretranslateWriterOptions); - - LatinWordTokenizer tokenizer = new(); - LatinWordDetokenizer detokenizer = new(); - ITruecaser truecaser = await _truecaserFactory.CreateAsync(engineDir, CancellationToken.None); - using IInteractiveTranslationModel smtModel = await _smtModelFactory.CreateAsync( - engineDir, - tokenizer, - detokenizer, - truecaser, - cancellationToken - ); - using ITranslationEngine? transferEngine = await _transferEngineFactory.CreateAsync( - engineDir, - tokenizer, - detokenizer, - truecaser, - cancellationToken - ); - HybridTranslationEngine hybridEngine = new(smtModel, transferEngine) { TargetDetokenizer = detokenizer }; - - await foreach (IReadOnlyList batch in BatchAsync(pretranslations)) - { - string[] segments = batch.Select(p => p.Translation).ToArray(); - IReadOnlyList results = await hybridEngine.TranslateBatchAsync( - segments, - cancellationToken - ); - foreach ((Pretranslation pretranslation, TranslationResult result) in batch.Zip(results)) - { - JsonSerializer.Serialize( - targetWriter, - pretranslation with - { - Translation = result.Translation - }, - JsonSerializerOptions - ); - } - } - } - - public static async IAsyncEnumerable> BatchAsync( - IAsyncEnumerable pretranslations - ) - { - List batch = []; - await foreach (Pretranslation item in pretranslations) - { - batch.Add(item); - if (batch.Count == BatchSize) - { - yield return batch; - batch = []; - } - } - if (batch.Count > 0) - yield return batch; - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/ThotSmtModelFactory.cs b/src/SIL.Machine.AspNetCore/Services/ThotSmtModelFactory.cs deleted file mode 100644 index 25b810a6c..000000000 --- a/src/SIL.Machine.AspNetCore/Services/ThotSmtModelFactory.cs +++ /dev/null @@ -1,110 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class ThotSmtModelFactory(IOptionsMonitor options) : ISmtModelFactory -{ - private readonly IOptionsMonitor _options = options; - - public Task CreateAsync( - string engineDir, - IRangeTokenizer tokenizer, - IDetokenizer detokenizer, - ITruecaser truecaser, - CancellationToken cancellationToken = default - ) - { - string smtConfigFileName = Path.Combine(engineDir, "smt.cfg"); - IInteractiveTranslationModel model = new ThotSmtModel(ThotWordAlignmentModelType.Hmm, smtConfigFileName) - { - SourceTokenizer = tokenizer, - TargetTokenizer = tokenizer, - TargetDetokenizer = detokenizer, - LowercaseSource = true, - LowercaseTarget = true, - Truecaser = truecaser - }; - return Task.FromResult(model); - } - - public Task CreateTrainerAsync( - string engineDir, - IRangeTokenizer tokenizer, - IParallelTextCorpus corpus, - CancellationToken cancellationToken = default - ) - { - string smtConfigFileName = Path.Combine(engineDir, "smt.cfg"); - ITrainer trainer = new ThotSmtModelTrainer(ThotWordAlignmentModelType.Hmm, corpus, smtConfigFileName) - { - SourceTokenizer = tokenizer, - TargetTokenizer = tokenizer, - LowercaseSource = true, - LowercaseTarget = true - }; - return Task.FromResult(trainer); - } - - public Task InitNewAsync(string engineDir, CancellationToken cancellationToken = default) - { - if (!Directory.Exists(engineDir)) - Directory.CreateDirectory(engineDir); - ZipFile.ExtractToDirectory(_options.CurrentValue.NewModelFile, engineDir); - return Task.CompletedTask; - } - - public Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default) - { - if (!Directory.Exists(engineDir)) - return Task.CompletedTask; - DirectoryHelper.DeleteDirectoryRobust(Path.Combine(engineDir, "lm")); - DirectoryHelper.DeleteDirectoryRobust(Path.Combine(engineDir, "tm")); - string smtConfigFileName = Path.Combine(engineDir, "smt.cfg"); - if (File.Exists(smtConfigFileName)) - File.Delete(smtConfigFileName); - if (!Directory.EnumerateFileSystemEntries(engineDir).Any()) - Directory.Delete(engineDir); - return Task.CompletedTask; - } - - public async Task UpdateEngineFromAsync( - string engineDir, - Stream source, - CancellationToken cancellationToken = default - ) - { - if (!Directory.Exists(engineDir)) - Directory.CreateDirectory(engineDir); - - await using MemoryStream memoryStream = new(); - await using (GZipStream gzipStream = new(source, CompressionMode.Decompress)) - { - await gzipStream.CopyToAsync(memoryStream, cancellationToken); - } - memoryStream.Seek(0, SeekOrigin.Begin); - await TarFile.ExtractToDirectoryAsync( - memoryStream, - engineDir, - overwriteFiles: true, - cancellationToken: cancellationToken - ); - } - - public async Task SaveEngineToAsync( - string engineDir, - Stream destination, - CancellationToken cancellationToken = default - ) - { - // create zip archive in memory stream - // This cannot be created directly to the shared stream because it all needs to be written at once - await using MemoryStream memoryStream = new(); - await TarFile.CreateFromDirectoryAsync( - engineDir, - memoryStream, - includeBaseDirectory: false, - cancellationToken: cancellationToken - ); - memoryStream.Seek(0, SeekOrigin.Begin); - await using GZipStream gzipStream = new(destination, CompressionMode.Compress); - await memoryStream.CopyToAsync(gzipStream, cancellationToken); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/TransferEngineFactory.cs b/src/SIL.Machine.AspNetCore/Services/TransferEngineFactory.cs deleted file mode 100644 index 300501b0b..000000000 --- a/src/SIL.Machine.AspNetCore/Services/TransferEngineFactory.cs +++ /dev/null @@ -1,61 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class TransferEngineFactory : ITransferEngineFactory -{ - public Task CreateAsync( - string engineDir, - IRangeTokenizer tokenizer, - IDetokenizer detokenizer, - ITruecaser truecaser, - CancellationToken cancellationToken = default - ) - { - string hcSrcConfigFileName = Path.Combine(engineDir, "src-hc.xml"); - string hcTrgConfigFileName = Path.Combine(engineDir, "trg-hc.xml"); - ITranslationEngine? transferEngine = null; - if (File.Exists(hcSrcConfigFileName) && File.Exists(hcTrgConfigFileName)) - { - var hcTraceManager = new TraceManager(); - - Language srcLang = XmlLanguageLoader.Load(hcSrcConfigFileName); - var srcMorpher = new Morpher(hcTraceManager, srcLang); - - Language trgLang = XmlLanguageLoader.Load(hcTrgConfigFileName); - var trgMorpher = new Morpher(hcTraceManager, trgLang); - - transferEngine = new TransferEngine( - srcMorpher, - new SimpleTransferer(new GlossMorphemeMapper(trgMorpher)), - trgMorpher - ) - { - SourceTokenizer = tokenizer, - TargetDetokenizer = detokenizer, - LowercaseSource = true, - Truecaser = truecaser - }; - } - return Task.FromResult(transferEngine); - } - - public Task InitNewAsync(string engineDir, CancellationToken cancellationToken = default) - { - // TODO: generate source and target config files - return Task.CompletedTask; - } - - public Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default) - { - if (!Directory.Exists(engineDir)) - return Task.CompletedTask; - string hcSrcConfigFileName = Path.Combine(engineDir, "src-hc.xml"); - if (File.Exists(hcSrcConfigFileName)) - File.Delete(hcSrcConfigFileName); - string hcTrgConfigFileName = Path.Combine(engineDir, "trg-hc.xml"); - if (File.Exists(hcTrgConfigFileName)) - File.Delete(hcTrgConfigFileName); - if (!Directory.EnumerateFileSystemEntries(engineDir).Any()) - Directory.Delete(engineDir); - return Task.CompletedTask; - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/TranslationEngineType.cs b/src/SIL.Machine.AspNetCore/Services/TranslationEngineType.cs deleted file mode 100644 index bda148cb7..000000000 --- a/src/SIL.Machine.AspNetCore/Services/TranslationEngineType.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public enum TranslationEngineType -{ - SmtTransfer, - Nmt -} diff --git a/src/SIL.Machine.AspNetCore/Services/UnigramTruecaserFactory.cs b/src/SIL.Machine.AspNetCore/Services/UnigramTruecaserFactory.cs deleted file mode 100644 index c5b8d70d3..000000000 --- a/src/SIL.Machine.AspNetCore/Services/UnigramTruecaserFactory.cs +++ /dev/null @@ -1,37 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class UnigramTruecaserFactory : ITruecaserFactory -{ - public async Task CreateAsync(string engineDir, CancellationToken cancellationToken = default) - { - var truecaser = new UnigramTruecaser(); - string path = GetModelPath(engineDir); - await truecaser.LoadAsync(path); - return truecaser; - } - - public Task CreateTrainerAsync( - string engineDir, - ITokenizer tokenizer, - ITextCorpus corpus, - CancellationToken cancellationToken = default - ) - { - string path = GetModelPath(engineDir); - ITrainer trainer = new UnigramTruecaserTrainer(path, corpus) { Tokenizer = tokenizer }; - return Task.FromResult(trainer); - } - - public Task CleanupAsync(string engineDir, CancellationToken cancellationToken = default) - { - string path = GetModelPath(engineDir); - if (File.Exists(path)) - File.Delete(path); - return Task.CompletedTask; - } - - private static string GetModelPath(string engineDir) - { - return Path.Combine(engineDir, "unigram-casing-model.txt"); - } -} diff --git a/src/SIL.Machine.AspNetCore/Services/UnimplementedInterceptor.cs b/src/SIL.Machine.AspNetCore/Services/UnimplementedInterceptor.cs deleted file mode 100644 index cbc8a9994..000000000 --- a/src/SIL.Machine.AspNetCore/Services/UnimplementedInterceptor.cs +++ /dev/null @@ -1,22 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -public class UnimplementedInterceptor : Interceptor -{ - public override async Task UnaryServerHandler( - TRequest request, - ServerCallContext context, - UnaryServerMethod continuation - ) - { - try - { - return await continuation(request, context); - } - catch (NotSupportedException) - { - throw new RpcException( - new Status(StatusCode.Unimplemented, "The call is not supported by the specified engine.") - ); - } - } -} diff --git a/src/SIL.Machine.AspNetCore/Usings.cs b/src/SIL.Machine.AspNetCore/Usings.cs deleted file mode 100644 index c1b0f091e..000000000 --- a/src/SIL.Machine.AspNetCore/Usings.cs +++ /dev/null @@ -1,59 +0,0 @@ -global using System.Collections.Concurrent; -global using System.Data; -global using System.Diagnostics; -global using System.Diagnostics.CodeAnalysis; -global using System.Formats.Tar; -global using System.Globalization; -global using System.IO.Compression; -global using System.Linq.Expressions; -global using System.Net; -global using System.Net.Mime; -global using System.Reflection; -global using System.Runtime.CompilerServices; -global using System.Security.Cryptography; -global using System.Text; -global using System.Text.Encodings.Web; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using System.Text.Json.Serialization; -global using System.Text.RegularExpressions; -global using Amazon; -global using Amazon.Runtime; -global using Amazon.S3; -global using Amazon.S3.Model; -global using CommunityToolkit.HighPerformance; -global using Grpc.Core; -global using Grpc.Core.Interceptors; -global using Grpc.Net.Client.Configuration; -global using Hangfire; -global using Hangfire.Common; -global using Hangfire.Mongo; -global using Hangfire.Mongo.Migration.Strategies; -global using Hangfire.Mongo.Migration.Strategies.Backup; -global using Hangfire.States; -global using Microsoft.AspNetCore.Routing; -global using Microsoft.Extensions.Configuration; -global using Microsoft.Extensions.DependencyInjection; -global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using MongoDB.Bson.Serialization.Serializers; -global using MongoDB.Driver; -global using MongoDB.Driver.Linq; -global using Nito.AsyncEx; -global using Nito.AsyncEx.Synchronous; -global using Polly; -global using SIL.DataAccess; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.AspNetCore.Services; -global using SIL.Machine.AspNetCore.Utils; -global using SIL.Machine.Corpora; -global using SIL.Machine.Morphology.HermitCrab; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Translation.Thot; -global using SIL.Machine.Utils; -global using SIL.Scripture; -global using SIL.WritingSystems; diff --git a/src/SIL.Machine.AspNetCore/Utils/AsyncDisposableBase.cs b/src/SIL.Machine.AspNetCore/Utils/AsyncDisposableBase.cs deleted file mode 100644 index b0f2d3bee..000000000 --- a/src/SIL.Machine.AspNetCore/Utils/AsyncDisposableBase.cs +++ /dev/null @@ -1,19 +0,0 @@ -using SIL.ObjectModel; - -namespace SIL.Machine.AspNetCore.Utils; - -public class AsyncDisposableBase : DisposableBase, IAsyncDisposable -{ - public async ValueTask DisposeAsync() - { - await DisposeAsyncCore(); - - Dispose(false); - GC.SuppressFinalize(this); - } - - protected virtual ValueTask DisposeAsyncCore() - { - return default; - } -} diff --git a/src/SIL.Machine.AspNetCore/Utils/AsyncTimer.cs b/src/SIL.Machine.AspNetCore/Utils/AsyncTimer.cs deleted file mode 100644 index 57d62bcbe..000000000 --- a/src/SIL.Machine.AspNetCore/Utils/AsyncTimer.cs +++ /dev/null @@ -1,69 +0,0 @@ -namespace SIL.Machine.AspNetCore.Utils; - -public class AsyncTimer : AsyncDisposableBase -{ - private readonly Timer _timer; - private readonly Func _callback; - private readonly AsyncLock _lock; - private bool _running; - - public AsyncTimer(Func callback) - { - _callback = callback; - _lock = new AsyncLock(); - _timer = new Timer(FireTimerAsync, null, Timeout.Infinite, Timeout.Infinite); - } - - public void Start(TimeSpan period) - { - _running = true; - _timer.Change(period, period); - } - - private async void FireTimerAsync(object? state) - { - using (await _lock.LockAsync()) - { - if (_running) - await _callback(); - } - } - - public async Task StopAsync() - { - using (await _lock.LockAsync()) - { - // FireTimer is *not* running _callback (since we got the lock) - StopTimer(); - } - // Now FireTimer will *never* run _callback - } - - public void Stop() - { - using (_lock.Lock()) - { - // FireTimer is *not* running _callback (since we got the lock) - StopTimer(); - } - // Now FireTimer will *never* run _callback - } - - private void StopTimer() - { - _timer.Change(Timeout.Infinite, Timeout.Infinite); - _running = false; - } - - protected override async ValueTask DisposeAsyncCore() - { - await StopAsync(); - _timer.Dispose(); - } - - protected override void DisposeManagedResources() - { - Stop(); - _timer.Dispose(); - } -} diff --git a/src/SIL.Machine.AspNetCore/Utils/CustomEnumConverterFactory.cs b/src/SIL.Machine.AspNetCore/Utils/CustomEnumConverterFactory.cs deleted file mode 100644 index b84d07b65..000000000 --- a/src/SIL.Machine.AspNetCore/Utils/CustomEnumConverterFactory.cs +++ /dev/null @@ -1,155 +0,0 @@ -namespace SIL.Machine.AspNetCore.Utils; - -public sealed class CustomEnumConverterFactory(JsonNamingPolicy namingPolicy) : JsonConverterFactory -{ - private readonly JsonNamingPolicy _namingPolicy = namingPolicy; - - public override bool CanConvert(Type typeToConvert) => typeToConvert.IsEnum; - - public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) - { - object[]? knownValues = null; - - if (typeToConvert == typeof(BindingFlags)) - { - knownValues = new object[] { BindingFlags.CreateInstance | BindingFlags.DeclaredOnly }; - } - - return (JsonConverter) - Activator.CreateInstance( - typeof(CustomEnumConverter<>).MakeGenericType(typeToConvert), - BindingFlags.Instance | BindingFlags.Public, - binder: null, - args: new object?[] { _namingPolicy, options, knownValues }, - culture: null - )!; - } -} - -public sealed class CustomEnumConverter : JsonConverter - where T : Enum -{ - private readonly JsonNamingPolicy _namingPolicy; - - private readonly Dictionary _readCache = new(); - private readonly Dictionary _writeCache = new(); - - // This converter will only support up to 64 enum values (including flags) on serialization and deserialization - private const int NameCacheLimit = 64; - - private const string ValueSeparator = ", "; - - public CustomEnumConverter(JsonNamingPolicy namingPolicy, JsonSerializerOptions options, object[]? knownValues) - { - _namingPolicy = namingPolicy; - - bool continueProcessing = true; - for (int i = 0; i < knownValues?.Length; i++) - { - if (!TryProcessValue((T)knownValues[i])) - { - continueProcessing = false; - break; - } - } - - if (continueProcessing) - { - Array values = Enum.GetValues(typeof(T)); - - for (int i = 0; i < values.Length; i++) - { - T value = (T)values.GetValue(i)!; - - if (!TryProcessValue(value)) - { - break; - } - } - } - - bool TryProcessValue(T value) - { - if (_readCache.Count == NameCacheLimit) - { - Debug.Assert(_writeCache.Count == NameCacheLimit); - return false; - } - - FormatAndAddToCaches(value, options.Encoder); - return true; - } - } - - public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - string? json; - - if ( - reader.TokenType != JsonTokenType.String - || (json = reader.GetString()) == null - || !_readCache.TryGetValue(json, out T? value) - ) - { - throw new JsonException(); - } - - return value; - } - - public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) - { - if (!_writeCache.TryGetValue(value, out JsonEncodedText formatted)) - { - if (_writeCache.Count == NameCacheLimit) - { - Debug.Assert(_readCache.Count == NameCacheLimit); - throw new ArgumentOutOfRangeException(); - } - - formatted = FormatAndAddToCaches(value, options.Encoder); - } - - writer.WriteStringValue(formatted); - } - - private JsonEncodedText FormatAndAddToCaches(T value, JavaScriptEncoder? encoder) - { - (string valueFormattedToStr, JsonEncodedText valueEncoded) = CustomEnumConverter.FormatEnumValue( - value.ToString(), - _namingPolicy, - encoder - ); - _readCache[valueFormattedToStr] = value; - _writeCache[value] = valueEncoded; - return valueEncoded; - } - - private static ValueTuple FormatEnumValue( - string value, - JsonNamingPolicy namingPolicy, - JavaScriptEncoder? encoder - ) - { - string converted; - - if (!value.Contains(ValueSeparator)) - { - converted = namingPolicy.ConvertName(value); - } - else - { - // todo: optimize implementation here by leveraging https://github.com/dotnet/runtime/issues/934. - string[] enumValues = value.Split(ValueSeparator); - - for (int i = 0; i < enumValues.Length; i++) - { - enumValues[i] = namingPolicy.ConvertName(enumValues[i]); - } - - converted = string.Join(ValueSeparator, enumValues); - } - - return (converted, JsonEncodedText.Encode(converted, encoder)); - } -} diff --git a/src/SIL.Machine.AspNetCore/Utils/DictionaryStringConverter.cs b/src/SIL.Machine.AspNetCore/Utils/DictionaryStringConverter.cs deleted file mode 100644 index 28e8eeac1..000000000 --- a/src/SIL.Machine.AspNetCore/Utils/DictionaryStringConverter.cs +++ /dev/null @@ -1,73 +0,0 @@ -namespace SIL.Machine.AspNetCore.Utils; - -internal sealed class DictionaryStringStringConverter : JsonConverter> -{ - public override IReadOnlyDictionary Read( - ref Utf8JsonReader reader, - Type typeToConvert, - JsonSerializerOptions options - ) - { - if (reader.TokenType != JsonTokenType.StartObject) - { - throw new JsonException($"JsonTokenType was of type {reader.TokenType}, only objects are supported"); - } - - var dictionary = new Dictionary(); - while (reader.Read()) - { - if (reader.TokenType == JsonTokenType.EndObject) - { - return dictionary; - } - - if (reader.TokenType != JsonTokenType.PropertyName) - { - throw new JsonException("JsonTokenType was not PropertyName"); - } - - var propertyName = reader.GetString(); - - if (string.IsNullOrWhiteSpace(propertyName)) - { - throw new JsonException("Failed to get property name"); - } - - reader.Read(); - - dictionary.Add(propertyName!, ExtractValue(ref reader)); - } - - return dictionary; - } - - public override void Write( - Utf8JsonWriter writer, - IReadOnlyDictionary value, - JsonSerializerOptions options - ) - { - JsonSerializer.Serialize(writer, value, options); - } - - private static string ExtractValue(ref Utf8JsonReader reader) - { - switch (reader.TokenType) - { - case JsonTokenType.String: - return reader.GetString() ?? "Error Reading String."; - case JsonTokenType.False: - return "false"; - case JsonTokenType.True: - return "true"; - case JsonTokenType.Null: - return "null"; - case JsonTokenType.Number: - if (reader.TryGetDouble(out var result)) - return result.ToString(CultureInfo.InvariantCulture); - return "Error Reading Number."; - default: - throw new JsonException($"'{reader.TokenType}' is not supported"); - } - } -} diff --git a/src/SIL.Machine.AspNetCore/Utils/EngineNotBuiltException.cs b/src/SIL.Machine.AspNetCore/Utils/EngineNotBuiltException.cs deleted file mode 100644 index dfc61a801..000000000 --- a/src/SIL.Machine.AspNetCore/Utils/EngineNotBuiltException.cs +++ /dev/null @@ -1,4 +0,0 @@ -namespace SIL.Machine.AspNetCore.Utils; - -/// This exception is thrown when an unbuilt engine is requested to perform an action that requires it being built -public class EngineNotBuiltException(string message) : Exception(message) { } diff --git a/src/SIL.Machine.AspNetCore/Utils/RecurrentTask.cs b/src/SIL.Machine.AspNetCore/Utils/RecurrentTask.cs deleted file mode 100644 index 35d806ef7..000000000 --- a/src/SIL.Machine.AspNetCore/Utils/RecurrentTask.cs +++ /dev/null @@ -1,40 +0,0 @@ -namespace SIL.Machine.AspNetCore.Utils; - -public abstract class RecurrentTask( - string serviceName, - IServiceProvider services, - TimeSpan period, - ILogger logger, - bool enable = true -) : BackgroundService -{ - private readonly bool _enable = enable; - private readonly string _serviceName = serviceName; - private readonly IServiceProvider _services = services; - private readonly TimeSpan _period = period; - private readonly ILogger _logger = logger; - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - if (!_enable) - return; - - using PeriodicTimer timer = new(_period); - - _logger.LogInformation($"{_serviceName} started."); - - try - { - while (await timer.WaitForNextTickAsync(stoppingToken)) - { - using IServiceScope scope = _services.CreateScope(); - await DoWorkAsync(scope, stoppingToken); - } - } - catch (OperationCanceledException) { } - - _logger.LogInformation($"{_serviceName} stopped."); - } - - protected abstract Task DoWorkAsync(IServiceScope scope, CancellationToken cancellationToken); -} diff --git a/src/SIL.Machine.AspNetCore/Utils/SharedFileUtils.cs b/src/SIL.Machine.AspNetCore/Utils/SharedFileUtils.cs deleted file mode 100644 index 78521f458..000000000 --- a/src/SIL.Machine.AspNetCore/Utils/SharedFileUtils.cs +++ /dev/null @@ -1,28 +0,0 @@ -namespace SIL.Machine.AspNetCore.Utils; - -public static class SharedFileUtils -{ - public static string Normalize(string path, bool includeLeadingSlash = false, bool includeTrailingSlash = false) - { - string normalizedPath = path; - if (normalizedPath == "/") - return normalizedPath; - if (!includeLeadingSlash && normalizedPath.StartsWith("/")) - { - normalizedPath = normalizedPath.Remove(0, 1); - } - else if (includeLeadingSlash && !normalizedPath.StartsWith("/")) - { - normalizedPath = "/" + normalizedPath; - } - if (!includeTrailingSlash && normalizedPath.EndsWith("/")) - { - normalizedPath = normalizedPath.Remove(normalizedPath.Length - 1, 1); - } - else if (includeTrailingSlash && !normalizedPath.EndsWith("/")) - { - normalizedPath += "/"; - } - return normalizedPath; - } -} diff --git a/src/SIL.Machine.AspNetCore/Utils/StartupTask.cs b/src/SIL.Machine.AspNetCore/Utils/StartupTask.cs deleted file mode 100644 index 47b19fd2c..000000000 --- a/src/SIL.Machine.AspNetCore/Utils/StartupTask.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace SIL.Machine.AspNetCore.Utils; - -public class StartupTask(IServiceProvider services, Func task) - : IHostedService -{ - private readonly IServiceProvider _services = services; - private readonly Func _task = task; - - public async Task StartAsync(CancellationToken cancellationToken) - { - using IServiceScope scope = _services.CreateScope(); - await _task(scope.ServiceProvider, cancellationToken); - } - - public Task StopAsync(CancellationToken cancellationToken) - { - return Task.CompletedTask; - } -} diff --git a/src/SIL.Machine.AspNetCore/data/.gitattributes b/src/SIL.Machine.AspNetCore/data/.gitattributes deleted file mode 100644 index be61261c7..000000000 --- a/src/SIL.Machine.AspNetCore/data/.gitattributes +++ /dev/null @@ -1,5 +0,0 @@ -thot-new-model/lm/* eol=lf -thot-new-model/tm/* eol=lf -thot-new-model/tm/*.hmm_alignd binary -thot-new-model/tm/*.hmm_lexnd binary -thot-new-model/smt.cfg eol=lf \ No newline at end of file diff --git a/src/SIL.Machine.AspNetCore/data/flores200languages.csv b/src/SIL.Machine.AspNetCore/data/flores200languages.csv deleted file mode 100644 index a97345541..000000000 --- a/src/SIL.Machine.AspNetCore/data/flores200languages.csv +++ /dev/null @@ -1,205 +0,0 @@ -language, code -Acehnese (Arabic script), ace_Arab -Acehnese (Latin script), ace_Latn -Mesopotamian Arabic, acm_Arab -Ta’izzi-Adeni Arabic, acq_Arab -Tunisian Arabic, aeb_Arab -Afrikaans, afr_Latn -South Levantine Arabic, ajp_Arab -Akan, aka_Latn -Amharic, amh_Ethi -North Levantine Arabic, apc_Arab -Modern Standard Arabic, arb_Arab -Modern Standard Arabic (Romanized), arb_Latn -Najdi Arabic, ars_Arab -Moroccan Arabic, ary_Arab -Egyptian Arabic, arz_Arab -Assamese, asm_Beng -Asturian, ast_Latn -Awadhi, awa_Deva -Central Aymara, ayr_Latn -South Azerbaijani, azb_Arab -North Azerbaijani, azj_Latn -Bashkir, bak_Cyrl -Bambara, bam_Latn -Balinese, ban_Latn -Belarusian, bel_Cyrl -Bemba, bem_Latn -Bengali, ben_Beng -Bhojpuri, bho_Deva -Banjar (Arabic script), bjn_Arab -Banjar (Latin script), bjn_Latn -Standard Tibetan, bod_Tibt -Bosnian, bos_Latn -Buginese, bug_Latn -Bulgarian, bul_Cyrl -Catalan, cat_Latn -Cebuano, ceb_Latn -Czech, ces_Latn -Chokwe, cjk_Latn -Central Kurdish, ckb_Arab -Crimean Tatar, crh_Latn -Welsh, cym_Latn -Danish, dan_Latn -German, deu_Latn -Southwestern Dinka, dik_Latn -Dyula, dyu_Latn -Dzongkha, dzo_Tibt -Greek, ell_Grek -English, eng_Latn -Esperanto, epo_Latn -Estonian, est_Latn -Basque, eus_Latn -Ewe, ewe_Latn -Faroese, fao_Latn -Fijian, fij_Latn -Finnish, fin_Latn -Fon, fon_Latn -French, fra_Latn -Friulian, fur_Latn -Nigerian Fulfulde, fuv_Latn -Scottish Gaelic, gla_Latn -Irish, gle_Latn -Galician, glg_Latn -Guarani, grn_Latn -Gujarati, guj_Gujr -Haitian Creole, hat_Latn -Hausa, hau_Latn -Hebrew, heb_Hebr -Hindi, hin_Deva -Chhattisgarhi, hne_Deva -Croatian, hrv_Latn -Hungarian, hun_Latn -Armenian, hye_Armn -Igbo, ibo_Latn -Ilocano, ilo_Latn -Indonesian, ind_Latn -Icelandic, isl_Latn -Italian, ita_Latn -Javanese, jav_Latn -Japanese, jpn_Jpan -Kabyle, kab_Latn -Jingpho, kac_Latn -Kamba, kam_Latn -Kannada, kan_Knda -Kashmiri (Arabic script), kas_Arab -Kashmiri (Devanagari script), kas_Deva -Georgian, kat_Geor -Central Kanuri (Arabic script), knc_Arab -Central Kanuri (Latin script), knc_Latn -Kazakh, kaz_Cyrl -Kabiyè, kbp_Latn -Kabuverdianu, kea_Latn -Khmer, khm_Khmr -Kikuyu, kik_Latn -Kinyarwanda, kin_Latn -Kyrgyz, kir_Cyrl -Kimbundu, kmb_Latn -Northern Kurdish, kmr_Latn -Kikongo, kon_Latn -Korean, kor_Hang -Lao, lao_Laoo -Ligurian, lij_Latn -Limburgish, lim_Latn -Lingala, lin_Latn -Lithuanian, lit_Latn -Lombard, lmo_Latn -Latgalian, ltg_Latn -Luxembourgish, ltz_Latn -Luba-Kasai, lua_Latn -Ganda, lug_Latn -Luo, luo_Latn -Mizo, lus_Latn -Standard Latvian, lvs_Latn -Magahi, mag_Deva -Maithili, mai_Deva -Malayalam, mal_Mlym -Marathi, mar_Deva -Minangkabau (Arabic script), min_Arab -Minangkabau (Latin script), min_Latn -Macedonian, mkd_Cyrl -Plateau Malagasy, plt_Latn -Maltese, mlt_Latn -Meitei (Bengali script), mni_Beng -Halh Mongolian, khk_Cyrl -Mossi, mos_Latn -Maori, mri_Latn -Burmese, mya_Mymr -Dutch, nld_Latn -Norwegian Nynorsk, nno_Latn -Norwegian Bokmål, nob_Latn -Nepali, npi_Deva -Northern Sotho, nso_Latn -Nuer, nus_Latn -Nyanja, nya_Latn -Occitan, oci_Latn -West Central Oromo, gaz_Latn -Odia, ory_Orya -Pangasinan, pag_Latn -Eastern Panjabi, pan_Guru -Papiamento, pap_Latn -Western Persian, pes_Arab -Polish, pol_Latn -Portuguese, por_Latn -Dari, prs_Arab -Southern Pashto, pbt_Arab -Ayacucho Quechua, quy_Latn -Romanian, ron_Latn -Rundi, run_Latn -Russian, rus_Cyrl -Sango, sag_Latn -Sanskrit, san_Deva -Santali, sat_Olck -Sicilian, scn_Latn -Shan, shn_Mymr -Sinhala, sin_Sinh -Slovak, slk_Latn -Slovenian, slv_Latn -Samoan, smo_Latn -Shona, sna_Latn -Sindhi, snd_Arab -Somali, som_Latn -Southern Sotho, sot_Latn -Spanish, spa_Latn -Tosk Albanian, als_Latn -Sardinian, srd_Latn -Serbian, srp_Cyrl -Swati, ssw_Latn -Sundanese, sun_Latn -Swedish, swe_Latn -Swahili, swh_Latn -Silesian, szl_Latn -Tamil, tam_Taml -Tatar, tat_Cyrl -Telugu, tel_Telu -Tajik, tgk_Cyrl -Tagalog, tgl_Latn -Thai, tha_Thai -Tigrinya, tir_Ethi -Tamasheq (Latin script), taq_Latn -Tamasheq (Tifinagh script), taq_Tfng -Tok Pisin, tpi_Latn -Tswana, tsn_Latn -Tsonga, tso_Latn -Turkmen, tuk_Latn -Tumbuka, tum_Latn -Turkish, tur_Latn -Twi, twi_Latn -Central Atlas Tamazight, tzm_Tfng -Uyghur, uig_Arab -Ukrainian, ukr_Cyrl -Umbundu, umb_Latn -Urdu, urd_Arab -Northern Uzbek, uzn_Latn -Venetian, vec_Latn -Vietnamese, vie_Latn -Waray, war_Latn -Wolof, wol_Latn -Xhosa, xho_Latn -Eastern Yiddish, ydd_Hebr -Yoruba, yor_Latn -Yue Chinese, yue_Hant -Chinese (Simplified), zho_Hans -Chinese (Traditional), zho_Hant -Standard Malay, zsm_Latn -Zulu, zul_Latn \ No newline at end of file diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/lm/trg.lm b/src/SIL.Machine.AspNetCore/data/thot-new-model/lm/trg.lm deleted file mode 100644 index 2bdf4838f..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/lm/trg.lm +++ /dev/null @@ -1,6 +0,0 @@ - 1.00000000 1.00000000 - 3.00000000 1.00000000 - 1.00000000 1.00000000 - 1.00000000 1.00000000 - 3.00000000 1.00000000 - 3.00000000 1.00000000 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/lm/trg.lm.weights b/src/SIL.Machine.AspNetCore/data/thot-new-model/lm/trg.lm.weights deleted file mode 100644 index 67a757d9d..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/lm/trg.lm.weights +++ /dev/null @@ -1 +0,0 @@ -3 3 10.000000 0.500000 0.500000 0.500000 0.500000 0.500000 0.500000 0.500000 0.500000 0.500000 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/lm/trg.lm.wp b/src/SIL.Machine.AspNetCore/data/thot-new-model/lm/trg.lm.wp deleted file mode 100644 index a3fd47b99..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/lm/trg.lm.wp +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/smt.cfg b/src/SIL.Machine.AspNetCore/data/thot-new-model/smt.cfg deleted file mode 100644 index b438a898f..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/smt.cfg +++ /dev/null @@ -1,29 +0,0 @@ -# Translation model prefix --tm tm/src_trg - -# Language model --lm lm/trg.lm - -# W parameter (maximum number of translation options to be considered per each source phrase) --W 10 - -# S parameter (maximum number of hypotheses that can be stored in each stack) --S 10 - -# A parameter (Maximum length in words of the source phrases to be translated) --A 7 - -# Degree of non-monotonicity --nomon 0 - -# Heuristic function used --h 6 - -# Best-first search flag --be - -# Translation model weights --tmw 0 0.5 1 1 1 1 0 1 - -# Set online learning parameters (ol_alg, lr_policy, l_stepsize, em_iters, e_par, r_par) --olp 0 0 1 5 1 0 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.lambda b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.lambda deleted file mode 100644 index 6e6566ce8..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.lambda +++ /dev/null @@ -1 +0,0 @@ -0.01 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.seglentable b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.seglentable deleted file mode 100644 index 7277bf4cf..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.seglentable +++ /dev/null @@ -1,10201 +0,0 @@ -0 0 1 -0 1 1 -0 2 1 -0 3 1 -0 4 1 -0 5 1 -0 6 1 -0 7 1 -0 8 1 -0 9 1 -0 10 1 -0 11 1 -0 12 1 -0 13 1 -0 14 1 -0 15 1 -0 16 1 -0 17 1 -0 18 1 -0 19 1 -0 20 1 -0 21 1 -0 22 1 -0 23 1 -0 24 1 -0 25 1 -0 26 1 -0 27 1 -0 28 1 -0 29 1 -0 30 1 -0 31 1 -0 32 1 -0 33 1 -0 34 1 -0 35 1 -0 36 1 -0 37 1 -0 38 1 -0 39 1 -0 40 1 -0 41 1 -0 42 1 -0 43 1 -0 44 1 -0 45 1 -0 46 1 -0 47 1 -0 48 1 -0 49 1 -0 50 1 -0 51 1 -0 52 1 -0 53 1 -0 54 1 -0 55 1 -0 56 1 -0 57 1 -0 58 1 -0 59 1 -0 60 1 -0 61 1 -0 62 1 -0 63 1 -0 64 1 -0 65 1 -0 66 1 -0 67 1 -0 68 1 -0 69 1 -0 70 1 -0 71 1 -0 72 1 -0 73 1 -0 74 1 -0 75 1 -0 76 1 -0 77 1 -0 78 1 -0 79 1 -0 80 1 -0 81 1 -0 82 1 -0 83 1 -0 84 1 -0 85 1 -0 86 1 -0 87 1 -0 88 1 -0 89 1 -0 90 1 -0 91 1 -0 92 1 -0 93 1 -0 94 1 -0 95 1 -0 96 1 -0 97 1 -0 98 1 -0 99 1 -0 100 1 -1 0 1 -1 1 1 -1 2 1 -1 3 1 -1 4 1 -1 5 1 -1 6 1 -1 7 1 -1 8 1 -1 9 1 -1 10 1 -1 11 1 -1 12 1 -1 13 1 -1 14 1 -1 15 1 -1 16 1 -1 17 1 -1 18 1 -1 19 1 -1 20 1 -1 21 1 -1 22 1 -1 23 1 -1 24 1 -1 25 1 -1 26 1 -1 27 1 -1 28 1 -1 29 1 -1 30 1 -1 31 1 -1 32 1 -1 33 1 -1 34 1 -1 35 1 -1 36 1 -1 37 1 -1 38 1 -1 39 1 -1 40 1 -1 41 1 -1 42 1 -1 43 1 -1 44 1 -1 45 1 -1 46 1 -1 47 1 -1 48 1 -1 49 1 -1 50 1 -1 51 1 -1 52 1 -1 53 1 -1 54 1 -1 55 1 -1 56 1 -1 57 1 -1 58 1 -1 59 1 -1 60 1 -1 61 1 -1 62 1 -1 63 1 -1 64 1 -1 65 1 -1 66 1 -1 67 1 -1 68 1 -1 69 1 -1 70 1 -1 71 1 -1 72 1 -1 73 1 -1 74 1 -1 75 1 -1 76 1 -1 77 1 -1 78 1 -1 79 1 -1 80 1 -1 81 1 -1 82 1 -1 83 1 -1 84 1 -1 85 1 -1 86 1 -1 87 1 -1 88 1 -1 89 1 -1 90 1 -1 91 1 -1 92 1 -1 93 1 -1 94 1 -1 95 1 -1 96 1 -1 97 1 -1 98 1 -1 99 1 -1 100 1 -2 0 1 -2 1 1 -2 2 1 -2 3 1 -2 4 1 -2 5 1 -2 6 1 -2 7 1 -2 8 1 -2 9 1 -2 10 1 -2 11 1 -2 12 1 -2 13 1 -2 14 1 -2 15 1 -2 16 1 -2 17 1 -2 18 1 -2 19 1 -2 20 1 -2 21 1 -2 22 1 -2 23 1 -2 24 1 -2 25 1 -2 26 1 -2 27 1 -2 28 1 -2 29 1 -2 30 1 -2 31 1 -2 32 1 -2 33 1 -2 34 1 -2 35 1 -2 36 1 -2 37 1 -2 38 1 -2 39 1 -2 40 1 -2 41 1 -2 42 1 -2 43 1 -2 44 1 -2 45 1 -2 46 1 -2 47 1 -2 48 1 -2 49 1 -2 50 1 -2 51 1 -2 52 1 -2 53 1 -2 54 1 -2 55 1 -2 56 1 -2 57 1 -2 58 1 -2 59 1 -2 60 1 -2 61 1 -2 62 1 -2 63 1 -2 64 1 -2 65 1 -2 66 1 -2 67 1 -2 68 1 -2 69 1 -2 70 1 -2 71 1 -2 72 1 -2 73 1 -2 74 1 -2 75 1 -2 76 1 -2 77 1 -2 78 1 -2 79 1 -2 80 1 -2 81 1 -2 82 1 -2 83 1 -2 84 1 -2 85 1 -2 86 1 -2 87 1 -2 88 1 -2 89 1 -2 90 1 -2 91 1 -2 92 1 -2 93 1 -2 94 1 -2 95 1 -2 96 1 -2 97 1 -2 98 1 -2 99 1 -2 100 1 -3 0 1 -3 1 1 -3 2 1 -3 3 1 -3 4 1 -3 5 1 -3 6 1 -3 7 1 -3 8 1 -3 9 1 -3 10 1 -3 11 1 -3 12 1 -3 13 1 -3 14 1 -3 15 1 -3 16 1 -3 17 1 -3 18 1 -3 19 1 -3 20 1 -3 21 1 -3 22 1 -3 23 1 -3 24 1 -3 25 1 -3 26 1 -3 27 1 -3 28 1 -3 29 1 -3 30 1 -3 31 1 -3 32 1 -3 33 1 -3 34 1 -3 35 1 -3 36 1 -3 37 1 -3 38 1 -3 39 1 -3 40 1 -3 41 1 -3 42 1 -3 43 1 -3 44 1 -3 45 1 -3 46 1 -3 47 1 -3 48 1 -3 49 1 -3 50 1 -3 51 1 -3 52 1 -3 53 1 -3 54 1 -3 55 1 -3 56 1 -3 57 1 -3 58 1 -3 59 1 -3 60 1 -3 61 1 -3 62 1 -3 63 1 -3 64 1 -3 65 1 -3 66 1 -3 67 1 -3 68 1 -3 69 1 -3 70 1 -3 71 1 -3 72 1 -3 73 1 -3 74 1 -3 75 1 -3 76 1 -3 77 1 -3 78 1 -3 79 1 -3 80 1 -3 81 1 -3 82 1 -3 83 1 -3 84 1 -3 85 1 -3 86 1 -3 87 1 -3 88 1 -3 89 1 -3 90 1 -3 91 1 -3 92 1 -3 93 1 -3 94 1 -3 95 1 -3 96 1 -3 97 1 -3 98 1 -3 99 1 -3 100 1 -4 0 1 -4 1 1 -4 2 1 -4 3 1 -4 4 1 -4 5 1 -4 6 1 -4 7 1 -4 8 1 -4 9 1 -4 10 1 -4 11 1 -4 12 1 -4 13 1 -4 14 1 -4 15 1 -4 16 1 -4 17 1 -4 18 1 -4 19 1 -4 20 1 -4 21 1 -4 22 1 -4 23 1 -4 24 1 -4 25 1 -4 26 1 -4 27 1 -4 28 1 -4 29 1 -4 30 1 -4 31 1 -4 32 1 -4 33 1 -4 34 1 -4 35 1 -4 36 1 -4 37 1 -4 38 1 -4 39 1 -4 40 1 -4 41 1 -4 42 1 -4 43 1 -4 44 1 -4 45 1 -4 46 1 -4 47 1 -4 48 1 -4 49 1 -4 50 1 -4 51 1 -4 52 1 -4 53 1 -4 54 1 -4 55 1 -4 56 1 -4 57 1 -4 58 1 -4 59 1 -4 60 1 -4 61 1 -4 62 1 -4 63 1 -4 64 1 -4 65 1 -4 66 1 -4 67 1 -4 68 1 -4 69 1 -4 70 1 -4 71 1 -4 72 1 -4 73 1 -4 74 1 -4 75 1 -4 76 1 -4 77 1 -4 78 1 -4 79 1 -4 80 1 -4 81 1 -4 82 1 -4 83 1 -4 84 1 -4 85 1 -4 86 1 -4 87 1 -4 88 1 -4 89 1 -4 90 1 -4 91 1 -4 92 1 -4 93 1 -4 94 1 -4 95 1 -4 96 1 -4 97 1 -4 98 1 -4 99 1 -4 100 1 -5 0 1 -5 1 1 -5 2 1 -5 3 1 -5 4 1 -5 5 1 -5 6 1 -5 7 1 -5 8 1 -5 9 1 -5 10 1 -5 11 1 -5 12 1 -5 13 1 -5 14 1 -5 15 1 -5 16 1 -5 17 1 -5 18 1 -5 19 1 -5 20 1 -5 21 1 -5 22 1 -5 23 1 -5 24 1 -5 25 1 -5 26 1 -5 27 1 -5 28 1 -5 29 1 -5 30 1 -5 31 1 -5 32 1 -5 33 1 -5 34 1 -5 35 1 -5 36 1 -5 37 1 -5 38 1 -5 39 1 -5 40 1 -5 41 1 -5 42 1 -5 43 1 -5 44 1 -5 45 1 -5 46 1 -5 47 1 -5 48 1 -5 49 1 -5 50 1 -5 51 1 -5 52 1 -5 53 1 -5 54 1 -5 55 1 -5 56 1 -5 57 1 -5 58 1 -5 59 1 -5 60 1 -5 61 1 -5 62 1 -5 63 1 -5 64 1 -5 65 1 -5 66 1 -5 67 1 -5 68 1 -5 69 1 -5 70 1 -5 71 1 -5 72 1 -5 73 1 -5 74 1 -5 75 1 -5 76 1 -5 77 1 -5 78 1 -5 79 1 -5 80 1 -5 81 1 -5 82 1 -5 83 1 -5 84 1 -5 85 1 -5 86 1 -5 87 1 -5 88 1 -5 89 1 -5 90 1 -5 91 1 -5 92 1 -5 93 1 -5 94 1 -5 95 1 -5 96 1 -5 97 1 -5 98 1 -5 99 1 -5 100 1 -6 0 1 -6 1 1 -6 2 1 -6 3 1 -6 4 1 -6 5 1 -6 6 1 -6 7 1 -6 8 1 -6 9 1 -6 10 1 -6 11 1 -6 12 1 -6 13 1 -6 14 1 -6 15 1 -6 16 1 -6 17 1 -6 18 1 -6 19 1 -6 20 1 -6 21 1 -6 22 1 -6 23 1 -6 24 1 -6 25 1 -6 26 1 -6 27 1 -6 28 1 -6 29 1 -6 30 1 -6 31 1 -6 32 1 -6 33 1 -6 34 1 -6 35 1 -6 36 1 -6 37 1 -6 38 1 -6 39 1 -6 40 1 -6 41 1 -6 42 1 -6 43 1 -6 44 1 -6 45 1 -6 46 1 -6 47 1 -6 48 1 -6 49 1 -6 50 1 -6 51 1 -6 52 1 -6 53 1 -6 54 1 -6 55 1 -6 56 1 -6 57 1 -6 58 1 -6 59 1 -6 60 1 -6 61 1 -6 62 1 -6 63 1 -6 64 1 -6 65 1 -6 66 1 -6 67 1 -6 68 1 -6 69 1 -6 70 1 -6 71 1 -6 72 1 -6 73 1 -6 74 1 -6 75 1 -6 76 1 -6 77 1 -6 78 1 -6 79 1 -6 80 1 -6 81 1 -6 82 1 -6 83 1 -6 84 1 -6 85 1 -6 86 1 -6 87 1 -6 88 1 -6 89 1 -6 90 1 -6 91 1 -6 92 1 -6 93 1 -6 94 1 -6 95 1 -6 96 1 -6 97 1 -6 98 1 -6 99 1 -6 100 1 -7 0 1 -7 1 1 -7 2 1 -7 3 1 -7 4 1 -7 5 1 -7 6 1 -7 7 1 -7 8 1 -7 9 1 -7 10 1 -7 11 1 -7 12 1 -7 13 1 -7 14 1 -7 15 1 -7 16 1 -7 17 1 -7 18 1 -7 19 1 -7 20 1 -7 21 1 -7 22 1 -7 23 1 -7 24 1 -7 25 1 -7 26 1 -7 27 1 -7 28 1 -7 29 1 -7 30 1 -7 31 1 -7 32 1 -7 33 1 -7 34 1 -7 35 1 -7 36 1 -7 37 1 -7 38 1 -7 39 1 -7 40 1 -7 41 1 -7 42 1 -7 43 1 -7 44 1 -7 45 1 -7 46 1 -7 47 1 -7 48 1 -7 49 1 -7 50 1 -7 51 1 -7 52 1 -7 53 1 -7 54 1 -7 55 1 -7 56 1 -7 57 1 -7 58 1 -7 59 1 -7 60 1 -7 61 1 -7 62 1 -7 63 1 -7 64 1 -7 65 1 -7 66 1 -7 67 1 -7 68 1 -7 69 1 -7 70 1 -7 71 1 -7 72 1 -7 73 1 -7 74 1 -7 75 1 -7 76 1 -7 77 1 -7 78 1 -7 79 1 -7 80 1 -7 81 1 -7 82 1 -7 83 1 -7 84 1 -7 85 1 -7 86 1 -7 87 1 -7 88 1 -7 89 1 -7 90 1 -7 91 1 -7 92 1 -7 93 1 -7 94 1 -7 95 1 -7 96 1 -7 97 1 -7 98 1 -7 99 1 -7 100 1 -8 0 1 -8 1 1 -8 2 1 -8 3 1 -8 4 1 -8 5 1 -8 6 1 -8 7 1 -8 8 1 -8 9 1 -8 10 1 -8 11 1 -8 12 1 -8 13 1 -8 14 1 -8 15 1 -8 16 1 -8 17 1 -8 18 1 -8 19 1 -8 20 1 -8 21 1 -8 22 1 -8 23 1 -8 24 1 -8 25 1 -8 26 1 -8 27 1 -8 28 1 -8 29 1 -8 30 1 -8 31 1 -8 32 1 -8 33 1 -8 34 1 -8 35 1 -8 36 1 -8 37 1 -8 38 1 -8 39 1 -8 40 1 -8 41 1 -8 42 1 -8 43 1 -8 44 1 -8 45 1 -8 46 1 -8 47 1 -8 48 1 -8 49 1 -8 50 1 -8 51 1 -8 52 1 -8 53 1 -8 54 1 -8 55 1 -8 56 1 -8 57 1 -8 58 1 -8 59 1 -8 60 1 -8 61 1 -8 62 1 -8 63 1 -8 64 1 -8 65 1 -8 66 1 -8 67 1 -8 68 1 -8 69 1 -8 70 1 -8 71 1 -8 72 1 -8 73 1 -8 74 1 -8 75 1 -8 76 1 -8 77 1 -8 78 1 -8 79 1 -8 80 1 -8 81 1 -8 82 1 -8 83 1 -8 84 1 -8 85 1 -8 86 1 -8 87 1 -8 88 1 -8 89 1 -8 90 1 -8 91 1 -8 92 1 -8 93 1 -8 94 1 -8 95 1 -8 96 1 -8 97 1 -8 98 1 -8 99 1 -8 100 1 -9 0 1 -9 1 1 -9 2 1 -9 3 1 -9 4 1 -9 5 1 -9 6 1 -9 7 1 -9 8 1 -9 9 1 -9 10 1 -9 11 1 -9 12 1 -9 13 1 -9 14 1 -9 15 1 -9 16 1 -9 17 1 -9 18 1 -9 19 1 -9 20 1 -9 21 1 -9 22 1 -9 23 1 -9 24 1 -9 25 1 -9 26 1 -9 27 1 -9 28 1 -9 29 1 -9 30 1 -9 31 1 -9 32 1 -9 33 1 -9 34 1 -9 35 1 -9 36 1 -9 37 1 -9 38 1 -9 39 1 -9 40 1 -9 41 1 -9 42 1 -9 43 1 -9 44 1 -9 45 1 -9 46 1 -9 47 1 -9 48 1 -9 49 1 -9 50 1 -9 51 1 -9 52 1 -9 53 1 -9 54 1 -9 55 1 -9 56 1 -9 57 1 -9 58 1 -9 59 1 -9 60 1 -9 61 1 -9 62 1 -9 63 1 -9 64 1 -9 65 1 -9 66 1 -9 67 1 -9 68 1 -9 69 1 -9 70 1 -9 71 1 -9 72 1 -9 73 1 -9 74 1 -9 75 1 -9 76 1 -9 77 1 -9 78 1 -9 79 1 -9 80 1 -9 81 1 -9 82 1 -9 83 1 -9 84 1 -9 85 1 -9 86 1 -9 87 1 -9 88 1 -9 89 1 -9 90 1 -9 91 1 -9 92 1 -9 93 1 -9 94 1 -9 95 1 -9 96 1 -9 97 1 -9 98 1 -9 99 1 -9 100 1 -10 0 1 -10 1 1 -10 2 1 -10 3 1 -10 4 1 -10 5 1 -10 6 1 -10 7 1 -10 8 1 -10 9 1 -10 10 1 -10 11 1 -10 12 1 -10 13 1 -10 14 1 -10 15 1 -10 16 1 -10 17 1 -10 18 1 -10 19 1 -10 20 1 -10 21 1 -10 22 1 -10 23 1 -10 24 1 -10 25 1 -10 26 1 -10 27 1 -10 28 1 -10 29 1 -10 30 1 -10 31 1 -10 32 1 -10 33 1 -10 34 1 -10 35 1 -10 36 1 -10 37 1 -10 38 1 -10 39 1 -10 40 1 -10 41 1 -10 42 1 -10 43 1 -10 44 1 -10 45 1 -10 46 1 -10 47 1 -10 48 1 -10 49 1 -10 50 1 -10 51 1 -10 52 1 -10 53 1 -10 54 1 -10 55 1 -10 56 1 -10 57 1 -10 58 1 -10 59 1 -10 60 1 -10 61 1 -10 62 1 -10 63 1 -10 64 1 -10 65 1 -10 66 1 -10 67 1 -10 68 1 -10 69 1 -10 70 1 -10 71 1 -10 72 1 -10 73 1 -10 74 1 -10 75 1 -10 76 1 -10 77 1 -10 78 1 -10 79 1 -10 80 1 -10 81 1 -10 82 1 -10 83 1 -10 84 1 -10 85 1 -10 86 1 -10 87 1 -10 88 1 -10 89 1 -10 90 1 -10 91 1 -10 92 1 -10 93 1 -10 94 1 -10 95 1 -10 96 1 -10 97 1 -10 98 1 -10 99 1 -10 100 1 -11 0 1 -11 1 1 -11 2 1 -11 3 1 -11 4 1 -11 5 1 -11 6 1 -11 7 1 -11 8 1 -11 9 1 -11 10 1 -11 11 1 -11 12 1 -11 13 1 -11 14 1 -11 15 1 -11 16 1 -11 17 1 -11 18 1 -11 19 1 -11 20 1 -11 21 1 -11 22 1 -11 23 1 -11 24 1 -11 25 1 -11 26 1 -11 27 1 -11 28 1 -11 29 1 -11 30 1 -11 31 1 -11 32 1 -11 33 1 -11 34 1 -11 35 1 -11 36 1 -11 37 1 -11 38 1 -11 39 1 -11 40 1 -11 41 1 -11 42 1 -11 43 1 -11 44 1 -11 45 1 -11 46 1 -11 47 1 -11 48 1 -11 49 1 -11 50 1 -11 51 1 -11 52 1 -11 53 1 -11 54 1 -11 55 1 -11 56 1 -11 57 1 -11 58 1 -11 59 1 -11 60 1 -11 61 1 -11 62 1 -11 63 1 -11 64 1 -11 65 1 -11 66 1 -11 67 1 -11 68 1 -11 69 1 -11 70 1 -11 71 1 -11 72 1 -11 73 1 -11 74 1 -11 75 1 -11 76 1 -11 77 1 -11 78 1 -11 79 1 -11 80 1 -11 81 1 -11 82 1 -11 83 1 -11 84 1 -11 85 1 -11 86 1 -11 87 1 -11 88 1 -11 89 1 -11 90 1 -11 91 1 -11 92 1 -11 93 1 -11 94 1 -11 95 1 -11 96 1 -11 97 1 -11 98 1 -11 99 1 -11 100 1 -12 0 1 -12 1 1 -12 2 1 -12 3 1 -12 4 1 -12 5 1 -12 6 1 -12 7 1 -12 8 1 -12 9 1 -12 10 1 -12 11 1 -12 12 1 -12 13 1 -12 14 1 -12 15 1 -12 16 1 -12 17 1 -12 18 1 -12 19 1 -12 20 1 -12 21 1 -12 22 1 -12 23 1 -12 24 1 -12 25 1 -12 26 1 -12 27 1 -12 28 1 -12 29 1 -12 30 1 -12 31 1 -12 32 1 -12 33 1 -12 34 1 -12 35 1 -12 36 1 -12 37 1 -12 38 1 -12 39 1 -12 40 1 -12 41 1 -12 42 1 -12 43 1 -12 44 1 -12 45 1 -12 46 1 -12 47 1 -12 48 1 -12 49 1 -12 50 1 -12 51 1 -12 52 1 -12 53 1 -12 54 1 -12 55 1 -12 56 1 -12 57 1 -12 58 1 -12 59 1 -12 60 1 -12 61 1 -12 62 1 -12 63 1 -12 64 1 -12 65 1 -12 66 1 -12 67 1 -12 68 1 -12 69 1 -12 70 1 -12 71 1 -12 72 1 -12 73 1 -12 74 1 -12 75 1 -12 76 1 -12 77 1 -12 78 1 -12 79 1 -12 80 1 -12 81 1 -12 82 1 -12 83 1 -12 84 1 -12 85 1 -12 86 1 -12 87 1 -12 88 1 -12 89 1 -12 90 1 -12 91 1 -12 92 1 -12 93 1 -12 94 1 -12 95 1 -12 96 1 -12 97 1 -12 98 1 -12 99 1 -12 100 1 -13 0 1 -13 1 1 -13 2 1 -13 3 1 -13 4 1 -13 5 1 -13 6 1 -13 7 1 -13 8 1 -13 9 1 -13 10 1 -13 11 1 -13 12 1 -13 13 1 -13 14 1 -13 15 1 -13 16 1 -13 17 1 -13 18 1 -13 19 1 -13 20 1 -13 21 1 -13 22 1 -13 23 1 -13 24 1 -13 25 1 -13 26 1 -13 27 1 -13 28 1 -13 29 1 -13 30 1 -13 31 1 -13 32 1 -13 33 1 -13 34 1 -13 35 1 -13 36 1 -13 37 1 -13 38 1 -13 39 1 -13 40 1 -13 41 1 -13 42 1 -13 43 1 -13 44 1 -13 45 1 -13 46 1 -13 47 1 -13 48 1 -13 49 1 -13 50 1 -13 51 1 -13 52 1 -13 53 1 -13 54 1 -13 55 1 -13 56 1 -13 57 1 -13 58 1 -13 59 1 -13 60 1 -13 61 1 -13 62 1 -13 63 1 -13 64 1 -13 65 1 -13 66 1 -13 67 1 -13 68 1 -13 69 1 -13 70 1 -13 71 1 -13 72 1 -13 73 1 -13 74 1 -13 75 1 -13 76 1 -13 77 1 -13 78 1 -13 79 1 -13 80 1 -13 81 1 -13 82 1 -13 83 1 -13 84 1 -13 85 1 -13 86 1 -13 87 1 -13 88 1 -13 89 1 -13 90 1 -13 91 1 -13 92 1 -13 93 1 -13 94 1 -13 95 1 -13 96 1 -13 97 1 -13 98 1 -13 99 1 -13 100 1 -14 0 1 -14 1 1 -14 2 1 -14 3 1 -14 4 1 -14 5 1 -14 6 1 -14 7 1 -14 8 1 -14 9 1 -14 10 1 -14 11 1 -14 12 1 -14 13 1 -14 14 1 -14 15 1 -14 16 1 -14 17 1 -14 18 1 -14 19 1 -14 20 1 -14 21 1 -14 22 1 -14 23 1 -14 24 1 -14 25 1 -14 26 1 -14 27 1 -14 28 1 -14 29 1 -14 30 1 -14 31 1 -14 32 1 -14 33 1 -14 34 1 -14 35 1 -14 36 1 -14 37 1 -14 38 1 -14 39 1 -14 40 1 -14 41 1 -14 42 1 -14 43 1 -14 44 1 -14 45 1 -14 46 1 -14 47 1 -14 48 1 -14 49 1 -14 50 1 -14 51 1 -14 52 1 -14 53 1 -14 54 1 -14 55 1 -14 56 1 -14 57 1 -14 58 1 -14 59 1 -14 60 1 -14 61 1 -14 62 1 -14 63 1 -14 64 1 -14 65 1 -14 66 1 -14 67 1 -14 68 1 -14 69 1 -14 70 1 -14 71 1 -14 72 1 -14 73 1 -14 74 1 -14 75 1 -14 76 1 -14 77 1 -14 78 1 -14 79 1 -14 80 1 -14 81 1 -14 82 1 -14 83 1 -14 84 1 -14 85 1 -14 86 1 -14 87 1 -14 88 1 -14 89 1 -14 90 1 -14 91 1 -14 92 1 -14 93 1 -14 94 1 -14 95 1 -14 96 1 -14 97 1 -14 98 1 -14 99 1 -14 100 1 -15 0 1 -15 1 1 -15 2 1 -15 3 1 -15 4 1 -15 5 1 -15 6 1 -15 7 1 -15 8 1 -15 9 1 -15 10 1 -15 11 1 -15 12 1 -15 13 1 -15 14 1 -15 15 1 -15 16 1 -15 17 1 -15 18 1 -15 19 1 -15 20 1 -15 21 1 -15 22 1 -15 23 1 -15 24 1 -15 25 1 -15 26 1 -15 27 1 -15 28 1 -15 29 1 -15 30 1 -15 31 1 -15 32 1 -15 33 1 -15 34 1 -15 35 1 -15 36 1 -15 37 1 -15 38 1 -15 39 1 -15 40 1 -15 41 1 -15 42 1 -15 43 1 -15 44 1 -15 45 1 -15 46 1 -15 47 1 -15 48 1 -15 49 1 -15 50 1 -15 51 1 -15 52 1 -15 53 1 -15 54 1 -15 55 1 -15 56 1 -15 57 1 -15 58 1 -15 59 1 -15 60 1 -15 61 1 -15 62 1 -15 63 1 -15 64 1 -15 65 1 -15 66 1 -15 67 1 -15 68 1 -15 69 1 -15 70 1 -15 71 1 -15 72 1 -15 73 1 -15 74 1 -15 75 1 -15 76 1 -15 77 1 -15 78 1 -15 79 1 -15 80 1 -15 81 1 -15 82 1 -15 83 1 -15 84 1 -15 85 1 -15 86 1 -15 87 1 -15 88 1 -15 89 1 -15 90 1 -15 91 1 -15 92 1 -15 93 1 -15 94 1 -15 95 1 -15 96 1 -15 97 1 -15 98 1 -15 99 1 -15 100 1 -16 0 1 -16 1 1 -16 2 1 -16 3 1 -16 4 1 -16 5 1 -16 6 1 -16 7 1 -16 8 1 -16 9 1 -16 10 1 -16 11 1 -16 12 1 -16 13 1 -16 14 1 -16 15 1 -16 16 1 -16 17 1 -16 18 1 -16 19 1 -16 20 1 -16 21 1 -16 22 1 -16 23 1 -16 24 1 -16 25 1 -16 26 1 -16 27 1 -16 28 1 -16 29 1 -16 30 1 -16 31 1 -16 32 1 -16 33 1 -16 34 1 -16 35 1 -16 36 1 -16 37 1 -16 38 1 -16 39 1 -16 40 1 -16 41 1 -16 42 1 -16 43 1 -16 44 1 -16 45 1 -16 46 1 -16 47 1 -16 48 1 -16 49 1 -16 50 1 -16 51 1 -16 52 1 -16 53 1 -16 54 1 -16 55 1 -16 56 1 -16 57 1 -16 58 1 -16 59 1 -16 60 1 -16 61 1 -16 62 1 -16 63 1 -16 64 1 -16 65 1 -16 66 1 -16 67 1 -16 68 1 -16 69 1 -16 70 1 -16 71 1 -16 72 1 -16 73 1 -16 74 1 -16 75 1 -16 76 1 -16 77 1 -16 78 1 -16 79 1 -16 80 1 -16 81 1 -16 82 1 -16 83 1 -16 84 1 -16 85 1 -16 86 1 -16 87 1 -16 88 1 -16 89 1 -16 90 1 -16 91 1 -16 92 1 -16 93 1 -16 94 1 -16 95 1 -16 96 1 -16 97 1 -16 98 1 -16 99 1 -16 100 1 -17 0 1 -17 1 1 -17 2 1 -17 3 1 -17 4 1 -17 5 1 -17 6 1 -17 7 1 -17 8 1 -17 9 1 -17 10 1 -17 11 1 -17 12 1 -17 13 1 -17 14 1 -17 15 1 -17 16 1 -17 17 1 -17 18 1 -17 19 1 -17 20 1 -17 21 1 -17 22 1 -17 23 1 -17 24 1 -17 25 1 -17 26 1 -17 27 1 -17 28 1 -17 29 1 -17 30 1 -17 31 1 -17 32 1 -17 33 1 -17 34 1 -17 35 1 -17 36 1 -17 37 1 -17 38 1 -17 39 1 -17 40 1 -17 41 1 -17 42 1 -17 43 1 -17 44 1 -17 45 1 -17 46 1 -17 47 1 -17 48 1 -17 49 1 -17 50 1 -17 51 1 -17 52 1 -17 53 1 -17 54 1 -17 55 1 -17 56 1 -17 57 1 -17 58 1 -17 59 1 -17 60 1 -17 61 1 -17 62 1 -17 63 1 -17 64 1 -17 65 1 -17 66 1 -17 67 1 -17 68 1 -17 69 1 -17 70 1 -17 71 1 -17 72 1 -17 73 1 -17 74 1 -17 75 1 -17 76 1 -17 77 1 -17 78 1 -17 79 1 -17 80 1 -17 81 1 -17 82 1 -17 83 1 -17 84 1 -17 85 1 -17 86 1 -17 87 1 -17 88 1 -17 89 1 -17 90 1 -17 91 1 -17 92 1 -17 93 1 -17 94 1 -17 95 1 -17 96 1 -17 97 1 -17 98 1 -17 99 1 -17 100 1 -18 0 1 -18 1 1 -18 2 1 -18 3 1 -18 4 1 -18 5 1 -18 6 1 -18 7 1 -18 8 1 -18 9 1 -18 10 1 -18 11 1 -18 12 1 -18 13 1 -18 14 1 -18 15 1 -18 16 1 -18 17 1 -18 18 1 -18 19 1 -18 20 1 -18 21 1 -18 22 1 -18 23 1 -18 24 1 -18 25 1 -18 26 1 -18 27 1 -18 28 1 -18 29 1 -18 30 1 -18 31 1 -18 32 1 -18 33 1 -18 34 1 -18 35 1 -18 36 1 -18 37 1 -18 38 1 -18 39 1 -18 40 1 -18 41 1 -18 42 1 -18 43 1 -18 44 1 -18 45 1 -18 46 1 -18 47 1 -18 48 1 -18 49 1 -18 50 1 -18 51 1 -18 52 1 -18 53 1 -18 54 1 -18 55 1 -18 56 1 -18 57 1 -18 58 1 -18 59 1 -18 60 1 -18 61 1 -18 62 1 -18 63 1 -18 64 1 -18 65 1 -18 66 1 -18 67 1 -18 68 1 -18 69 1 -18 70 1 -18 71 1 -18 72 1 -18 73 1 -18 74 1 -18 75 1 -18 76 1 -18 77 1 -18 78 1 -18 79 1 -18 80 1 -18 81 1 -18 82 1 -18 83 1 -18 84 1 -18 85 1 -18 86 1 -18 87 1 -18 88 1 -18 89 1 -18 90 1 -18 91 1 -18 92 1 -18 93 1 -18 94 1 -18 95 1 -18 96 1 -18 97 1 -18 98 1 -18 99 1 -18 100 1 -19 0 1 -19 1 1 -19 2 1 -19 3 1 -19 4 1 -19 5 1 -19 6 1 -19 7 1 -19 8 1 -19 9 1 -19 10 1 -19 11 1 -19 12 1 -19 13 1 -19 14 1 -19 15 1 -19 16 1 -19 17 1 -19 18 1 -19 19 1 -19 20 1 -19 21 1 -19 22 1 -19 23 1 -19 24 1 -19 25 1 -19 26 1 -19 27 1 -19 28 1 -19 29 1 -19 30 1 -19 31 1 -19 32 1 -19 33 1 -19 34 1 -19 35 1 -19 36 1 -19 37 1 -19 38 1 -19 39 1 -19 40 1 -19 41 1 -19 42 1 -19 43 1 -19 44 1 -19 45 1 -19 46 1 -19 47 1 -19 48 1 -19 49 1 -19 50 1 -19 51 1 -19 52 1 -19 53 1 -19 54 1 -19 55 1 -19 56 1 -19 57 1 -19 58 1 -19 59 1 -19 60 1 -19 61 1 -19 62 1 -19 63 1 -19 64 1 -19 65 1 -19 66 1 -19 67 1 -19 68 1 -19 69 1 -19 70 1 -19 71 1 -19 72 1 -19 73 1 -19 74 1 -19 75 1 -19 76 1 -19 77 1 -19 78 1 -19 79 1 -19 80 1 -19 81 1 -19 82 1 -19 83 1 -19 84 1 -19 85 1 -19 86 1 -19 87 1 -19 88 1 -19 89 1 -19 90 1 -19 91 1 -19 92 1 -19 93 1 -19 94 1 -19 95 1 -19 96 1 -19 97 1 -19 98 1 -19 99 1 -19 100 1 -20 0 1 -20 1 1 -20 2 1 -20 3 1 -20 4 1 -20 5 1 -20 6 1 -20 7 1 -20 8 1 -20 9 1 -20 10 1 -20 11 1 -20 12 1 -20 13 1 -20 14 1 -20 15 1 -20 16 1 -20 17 1 -20 18 1 -20 19 1 -20 20 1 -20 21 1 -20 22 1 -20 23 1 -20 24 1 -20 25 1 -20 26 1 -20 27 1 -20 28 1 -20 29 1 -20 30 1 -20 31 1 -20 32 1 -20 33 1 -20 34 1 -20 35 1 -20 36 1 -20 37 1 -20 38 1 -20 39 1 -20 40 1 -20 41 1 -20 42 1 -20 43 1 -20 44 1 -20 45 1 -20 46 1 -20 47 1 -20 48 1 -20 49 1 -20 50 1 -20 51 1 -20 52 1 -20 53 1 -20 54 1 -20 55 1 -20 56 1 -20 57 1 -20 58 1 -20 59 1 -20 60 1 -20 61 1 -20 62 1 -20 63 1 -20 64 1 -20 65 1 -20 66 1 -20 67 1 -20 68 1 -20 69 1 -20 70 1 -20 71 1 -20 72 1 -20 73 1 -20 74 1 -20 75 1 -20 76 1 -20 77 1 -20 78 1 -20 79 1 -20 80 1 -20 81 1 -20 82 1 -20 83 1 -20 84 1 -20 85 1 -20 86 1 -20 87 1 -20 88 1 -20 89 1 -20 90 1 -20 91 1 -20 92 1 -20 93 1 -20 94 1 -20 95 1 -20 96 1 -20 97 1 -20 98 1 -20 99 1 -20 100 1 -21 0 1 -21 1 1 -21 2 1 -21 3 1 -21 4 1 -21 5 1 -21 6 1 -21 7 1 -21 8 1 -21 9 1 -21 10 1 -21 11 1 -21 12 1 -21 13 1 -21 14 1 -21 15 1 -21 16 1 -21 17 1 -21 18 1 -21 19 1 -21 20 1 -21 21 1 -21 22 1 -21 23 1 -21 24 1 -21 25 1 -21 26 1 -21 27 1 -21 28 1 -21 29 1 -21 30 1 -21 31 1 -21 32 1 -21 33 1 -21 34 1 -21 35 1 -21 36 1 -21 37 1 -21 38 1 -21 39 1 -21 40 1 -21 41 1 -21 42 1 -21 43 1 -21 44 1 -21 45 1 -21 46 1 -21 47 1 -21 48 1 -21 49 1 -21 50 1 -21 51 1 -21 52 1 -21 53 1 -21 54 1 -21 55 1 -21 56 1 -21 57 1 -21 58 1 -21 59 1 -21 60 1 -21 61 1 -21 62 1 -21 63 1 -21 64 1 -21 65 1 -21 66 1 -21 67 1 -21 68 1 -21 69 1 -21 70 1 -21 71 1 -21 72 1 -21 73 1 -21 74 1 -21 75 1 -21 76 1 -21 77 1 -21 78 1 -21 79 1 -21 80 1 -21 81 1 -21 82 1 -21 83 1 -21 84 1 -21 85 1 -21 86 1 -21 87 1 -21 88 1 -21 89 1 -21 90 1 -21 91 1 -21 92 1 -21 93 1 -21 94 1 -21 95 1 -21 96 1 -21 97 1 -21 98 1 -21 99 1 -21 100 1 -22 0 1 -22 1 1 -22 2 1 -22 3 1 -22 4 1 -22 5 1 -22 6 1 -22 7 1 -22 8 1 -22 9 1 -22 10 1 -22 11 1 -22 12 1 -22 13 1 -22 14 1 -22 15 1 -22 16 1 -22 17 1 -22 18 1 -22 19 1 -22 20 1 -22 21 1 -22 22 1 -22 23 1 -22 24 1 -22 25 1 -22 26 1 -22 27 1 -22 28 1 -22 29 1 -22 30 1 -22 31 1 -22 32 1 -22 33 1 -22 34 1 -22 35 1 -22 36 1 -22 37 1 -22 38 1 -22 39 1 -22 40 1 -22 41 1 -22 42 1 -22 43 1 -22 44 1 -22 45 1 -22 46 1 -22 47 1 -22 48 1 -22 49 1 -22 50 1 -22 51 1 -22 52 1 -22 53 1 -22 54 1 -22 55 1 -22 56 1 -22 57 1 -22 58 1 -22 59 1 -22 60 1 -22 61 1 -22 62 1 -22 63 1 -22 64 1 -22 65 1 -22 66 1 -22 67 1 -22 68 1 -22 69 1 -22 70 1 -22 71 1 -22 72 1 -22 73 1 -22 74 1 -22 75 1 -22 76 1 -22 77 1 -22 78 1 -22 79 1 -22 80 1 -22 81 1 -22 82 1 -22 83 1 -22 84 1 -22 85 1 -22 86 1 -22 87 1 -22 88 1 -22 89 1 -22 90 1 -22 91 1 -22 92 1 -22 93 1 -22 94 1 -22 95 1 -22 96 1 -22 97 1 -22 98 1 -22 99 1 -22 100 1 -23 0 1 -23 1 1 -23 2 1 -23 3 1 -23 4 1 -23 5 1 -23 6 1 -23 7 1 -23 8 1 -23 9 1 -23 10 1 -23 11 1 -23 12 1 -23 13 1 -23 14 1 -23 15 1 -23 16 1 -23 17 1 -23 18 1 -23 19 1 -23 20 1 -23 21 1 -23 22 1 -23 23 1 -23 24 1 -23 25 1 -23 26 1 -23 27 1 -23 28 1 -23 29 1 -23 30 1 -23 31 1 -23 32 1 -23 33 1 -23 34 1 -23 35 1 -23 36 1 -23 37 1 -23 38 1 -23 39 1 -23 40 1 -23 41 1 -23 42 1 -23 43 1 -23 44 1 -23 45 1 -23 46 1 -23 47 1 -23 48 1 -23 49 1 -23 50 1 -23 51 1 -23 52 1 -23 53 1 -23 54 1 -23 55 1 -23 56 1 -23 57 1 -23 58 1 -23 59 1 -23 60 1 -23 61 1 -23 62 1 -23 63 1 -23 64 1 -23 65 1 -23 66 1 -23 67 1 -23 68 1 -23 69 1 -23 70 1 -23 71 1 -23 72 1 -23 73 1 -23 74 1 -23 75 1 -23 76 1 -23 77 1 -23 78 1 -23 79 1 -23 80 1 -23 81 1 -23 82 1 -23 83 1 -23 84 1 -23 85 1 -23 86 1 -23 87 1 -23 88 1 -23 89 1 -23 90 1 -23 91 1 -23 92 1 -23 93 1 -23 94 1 -23 95 1 -23 96 1 -23 97 1 -23 98 1 -23 99 1 -23 100 1 -24 0 1 -24 1 1 -24 2 1 -24 3 1 -24 4 1 -24 5 1 -24 6 1 -24 7 1 -24 8 1 -24 9 1 -24 10 1 -24 11 1 -24 12 1 -24 13 1 -24 14 1 -24 15 1 -24 16 1 -24 17 1 -24 18 1 -24 19 1 -24 20 1 -24 21 1 -24 22 1 -24 23 1 -24 24 1 -24 25 1 -24 26 1 -24 27 1 -24 28 1 -24 29 1 -24 30 1 -24 31 1 -24 32 1 -24 33 1 -24 34 1 -24 35 1 -24 36 1 -24 37 1 -24 38 1 -24 39 1 -24 40 1 -24 41 1 -24 42 1 -24 43 1 -24 44 1 -24 45 1 -24 46 1 -24 47 1 -24 48 1 -24 49 1 -24 50 1 -24 51 1 -24 52 1 -24 53 1 -24 54 1 -24 55 1 -24 56 1 -24 57 1 -24 58 1 -24 59 1 -24 60 1 -24 61 1 -24 62 1 -24 63 1 -24 64 1 -24 65 1 -24 66 1 -24 67 1 -24 68 1 -24 69 1 -24 70 1 -24 71 1 -24 72 1 -24 73 1 -24 74 1 -24 75 1 -24 76 1 -24 77 1 -24 78 1 -24 79 1 -24 80 1 -24 81 1 -24 82 1 -24 83 1 -24 84 1 -24 85 1 -24 86 1 -24 87 1 -24 88 1 -24 89 1 -24 90 1 -24 91 1 -24 92 1 -24 93 1 -24 94 1 -24 95 1 -24 96 1 -24 97 1 -24 98 1 -24 99 1 -24 100 1 -25 0 1 -25 1 1 -25 2 1 -25 3 1 -25 4 1 -25 5 1 -25 6 1 -25 7 1 -25 8 1 -25 9 1 -25 10 1 -25 11 1 -25 12 1 -25 13 1 -25 14 1 -25 15 1 -25 16 1 -25 17 1 -25 18 1 -25 19 1 -25 20 1 -25 21 1 -25 22 1 -25 23 1 -25 24 1 -25 25 1 -25 26 1 -25 27 1 -25 28 1 -25 29 1 -25 30 1 -25 31 1 -25 32 1 -25 33 1 -25 34 1 -25 35 1 -25 36 1 -25 37 1 -25 38 1 -25 39 1 -25 40 1 -25 41 1 -25 42 1 -25 43 1 -25 44 1 -25 45 1 -25 46 1 -25 47 1 -25 48 1 -25 49 1 -25 50 1 -25 51 1 -25 52 1 -25 53 1 -25 54 1 -25 55 1 -25 56 1 -25 57 1 -25 58 1 -25 59 1 -25 60 1 -25 61 1 -25 62 1 -25 63 1 -25 64 1 -25 65 1 -25 66 1 -25 67 1 -25 68 1 -25 69 1 -25 70 1 -25 71 1 -25 72 1 -25 73 1 -25 74 1 -25 75 1 -25 76 1 -25 77 1 -25 78 1 -25 79 1 -25 80 1 -25 81 1 -25 82 1 -25 83 1 -25 84 1 -25 85 1 -25 86 1 -25 87 1 -25 88 1 -25 89 1 -25 90 1 -25 91 1 -25 92 1 -25 93 1 -25 94 1 -25 95 1 -25 96 1 -25 97 1 -25 98 1 -25 99 1 -25 100 1 -26 0 1 -26 1 1 -26 2 1 -26 3 1 -26 4 1 -26 5 1 -26 6 1 -26 7 1 -26 8 1 -26 9 1 -26 10 1 -26 11 1 -26 12 1 -26 13 1 -26 14 1 -26 15 1 -26 16 1 -26 17 1 -26 18 1 -26 19 1 -26 20 1 -26 21 1 -26 22 1 -26 23 1 -26 24 1 -26 25 1 -26 26 1 -26 27 1 -26 28 1 -26 29 1 -26 30 1 -26 31 1 -26 32 1 -26 33 1 -26 34 1 -26 35 1 -26 36 1 -26 37 1 -26 38 1 -26 39 1 -26 40 1 -26 41 1 -26 42 1 -26 43 1 -26 44 1 -26 45 1 -26 46 1 -26 47 1 -26 48 1 -26 49 1 -26 50 1 -26 51 1 -26 52 1 -26 53 1 -26 54 1 -26 55 1 -26 56 1 -26 57 1 -26 58 1 -26 59 1 -26 60 1 -26 61 1 -26 62 1 -26 63 1 -26 64 1 -26 65 1 -26 66 1 -26 67 1 -26 68 1 -26 69 1 -26 70 1 -26 71 1 -26 72 1 -26 73 1 -26 74 1 -26 75 1 -26 76 1 -26 77 1 -26 78 1 -26 79 1 -26 80 1 -26 81 1 -26 82 1 -26 83 1 -26 84 1 -26 85 1 -26 86 1 -26 87 1 -26 88 1 -26 89 1 -26 90 1 -26 91 1 -26 92 1 -26 93 1 -26 94 1 -26 95 1 -26 96 1 -26 97 1 -26 98 1 -26 99 1 -26 100 1 -27 0 1 -27 1 1 -27 2 1 -27 3 1 -27 4 1 -27 5 1 -27 6 1 -27 7 1 -27 8 1 -27 9 1 -27 10 1 -27 11 1 -27 12 1 -27 13 1 -27 14 1 -27 15 1 -27 16 1 -27 17 1 -27 18 1 -27 19 1 -27 20 1 -27 21 1 -27 22 1 -27 23 1 -27 24 1 -27 25 1 -27 26 1 -27 27 1 -27 28 1 -27 29 1 -27 30 1 -27 31 1 -27 32 1 -27 33 1 -27 34 1 -27 35 1 -27 36 1 -27 37 1 -27 38 1 -27 39 1 -27 40 1 -27 41 1 -27 42 1 -27 43 1 -27 44 1 -27 45 1 -27 46 1 -27 47 1 -27 48 1 -27 49 1 -27 50 1 -27 51 1 -27 52 1 -27 53 1 -27 54 1 -27 55 1 -27 56 1 -27 57 1 -27 58 1 -27 59 1 -27 60 1 -27 61 1 -27 62 1 -27 63 1 -27 64 1 -27 65 1 -27 66 1 -27 67 1 -27 68 1 -27 69 1 -27 70 1 -27 71 1 -27 72 1 -27 73 1 -27 74 1 -27 75 1 -27 76 1 -27 77 1 -27 78 1 -27 79 1 -27 80 1 -27 81 1 -27 82 1 -27 83 1 -27 84 1 -27 85 1 -27 86 1 -27 87 1 -27 88 1 -27 89 1 -27 90 1 -27 91 1 -27 92 1 -27 93 1 -27 94 1 -27 95 1 -27 96 1 -27 97 1 -27 98 1 -27 99 1 -27 100 1 -28 0 1 -28 1 1 -28 2 1 -28 3 1 -28 4 1 -28 5 1 -28 6 1 -28 7 1 -28 8 1 -28 9 1 -28 10 1 -28 11 1 -28 12 1 -28 13 1 -28 14 1 -28 15 1 -28 16 1 -28 17 1 -28 18 1 -28 19 1 -28 20 1 -28 21 1 -28 22 1 -28 23 1 -28 24 1 -28 25 1 -28 26 1 -28 27 1 -28 28 1 -28 29 1 -28 30 1 -28 31 1 -28 32 1 -28 33 1 -28 34 1 -28 35 1 -28 36 1 -28 37 1 -28 38 1 -28 39 1 -28 40 1 -28 41 1 -28 42 1 -28 43 1 -28 44 1 -28 45 1 -28 46 1 -28 47 1 -28 48 1 -28 49 1 -28 50 1 -28 51 1 -28 52 1 -28 53 1 -28 54 1 -28 55 1 -28 56 1 -28 57 1 -28 58 1 -28 59 1 -28 60 1 -28 61 1 -28 62 1 -28 63 1 -28 64 1 -28 65 1 -28 66 1 -28 67 1 -28 68 1 -28 69 1 -28 70 1 -28 71 1 -28 72 1 -28 73 1 -28 74 1 -28 75 1 -28 76 1 -28 77 1 -28 78 1 -28 79 1 -28 80 1 -28 81 1 -28 82 1 -28 83 1 -28 84 1 -28 85 1 -28 86 1 -28 87 1 -28 88 1 -28 89 1 -28 90 1 -28 91 1 -28 92 1 -28 93 1 -28 94 1 -28 95 1 -28 96 1 -28 97 1 -28 98 1 -28 99 1 -28 100 1 -29 0 1 -29 1 1 -29 2 1 -29 3 1 -29 4 1 -29 5 1 -29 6 1 -29 7 1 -29 8 1 -29 9 1 -29 10 1 -29 11 1 -29 12 1 -29 13 1 -29 14 1 -29 15 1 -29 16 1 -29 17 1 -29 18 1 -29 19 1 -29 20 1 -29 21 1 -29 22 1 -29 23 1 -29 24 1 -29 25 1 -29 26 1 -29 27 1 -29 28 1 -29 29 1 -29 30 1 -29 31 1 -29 32 1 -29 33 1 -29 34 1 -29 35 1 -29 36 1 -29 37 1 -29 38 1 -29 39 1 -29 40 1 -29 41 1 -29 42 1 -29 43 1 -29 44 1 -29 45 1 -29 46 1 -29 47 1 -29 48 1 -29 49 1 -29 50 1 -29 51 1 -29 52 1 -29 53 1 -29 54 1 -29 55 1 -29 56 1 -29 57 1 -29 58 1 -29 59 1 -29 60 1 -29 61 1 -29 62 1 -29 63 1 -29 64 1 -29 65 1 -29 66 1 -29 67 1 -29 68 1 -29 69 1 -29 70 1 -29 71 1 -29 72 1 -29 73 1 -29 74 1 -29 75 1 -29 76 1 -29 77 1 -29 78 1 -29 79 1 -29 80 1 -29 81 1 -29 82 1 -29 83 1 -29 84 1 -29 85 1 -29 86 1 -29 87 1 -29 88 1 -29 89 1 -29 90 1 -29 91 1 -29 92 1 -29 93 1 -29 94 1 -29 95 1 -29 96 1 -29 97 1 -29 98 1 -29 99 1 -29 100 1 -30 0 1 -30 1 1 -30 2 1 -30 3 1 -30 4 1 -30 5 1 -30 6 1 -30 7 1 -30 8 1 -30 9 1 -30 10 1 -30 11 1 -30 12 1 -30 13 1 -30 14 1 -30 15 1 -30 16 1 -30 17 1 -30 18 1 -30 19 1 -30 20 1 -30 21 1 -30 22 1 -30 23 1 -30 24 1 -30 25 1 -30 26 1 -30 27 1 -30 28 1 -30 29 1 -30 30 1 -30 31 1 -30 32 1 -30 33 1 -30 34 1 -30 35 1 -30 36 1 -30 37 1 -30 38 1 -30 39 1 -30 40 1 -30 41 1 -30 42 1 -30 43 1 -30 44 1 -30 45 1 -30 46 1 -30 47 1 -30 48 1 -30 49 1 -30 50 1 -30 51 1 -30 52 1 -30 53 1 -30 54 1 -30 55 1 -30 56 1 -30 57 1 -30 58 1 -30 59 1 -30 60 1 -30 61 1 -30 62 1 -30 63 1 -30 64 1 -30 65 1 -30 66 1 -30 67 1 -30 68 1 -30 69 1 -30 70 1 -30 71 1 -30 72 1 -30 73 1 -30 74 1 -30 75 1 -30 76 1 -30 77 1 -30 78 1 -30 79 1 -30 80 1 -30 81 1 -30 82 1 -30 83 1 -30 84 1 -30 85 1 -30 86 1 -30 87 1 -30 88 1 -30 89 1 -30 90 1 -30 91 1 -30 92 1 -30 93 1 -30 94 1 -30 95 1 -30 96 1 -30 97 1 -30 98 1 -30 99 1 -30 100 1 -31 0 1 -31 1 1 -31 2 1 -31 3 1 -31 4 1 -31 5 1 -31 6 1 -31 7 1 -31 8 1 -31 9 1 -31 10 1 -31 11 1 -31 12 1 -31 13 1 -31 14 1 -31 15 1 -31 16 1 -31 17 1 -31 18 1 -31 19 1 -31 20 1 -31 21 1 -31 22 1 -31 23 1 -31 24 1 -31 25 1 -31 26 1 -31 27 1 -31 28 1 -31 29 1 -31 30 1 -31 31 1 -31 32 1 -31 33 1 -31 34 1 -31 35 1 -31 36 1 -31 37 1 -31 38 1 -31 39 1 -31 40 1 -31 41 1 -31 42 1 -31 43 1 -31 44 1 -31 45 1 -31 46 1 -31 47 1 -31 48 1 -31 49 1 -31 50 1 -31 51 1 -31 52 1 -31 53 1 -31 54 1 -31 55 1 -31 56 1 -31 57 1 -31 58 1 -31 59 1 -31 60 1 -31 61 1 -31 62 1 -31 63 1 -31 64 1 -31 65 1 -31 66 1 -31 67 1 -31 68 1 -31 69 1 -31 70 1 -31 71 1 -31 72 1 -31 73 1 -31 74 1 -31 75 1 -31 76 1 -31 77 1 -31 78 1 -31 79 1 -31 80 1 -31 81 1 -31 82 1 -31 83 1 -31 84 1 -31 85 1 -31 86 1 -31 87 1 -31 88 1 -31 89 1 -31 90 1 -31 91 1 -31 92 1 -31 93 1 -31 94 1 -31 95 1 -31 96 1 -31 97 1 -31 98 1 -31 99 1 -31 100 1 -32 0 1 -32 1 1 -32 2 1 -32 3 1 -32 4 1 -32 5 1 -32 6 1 -32 7 1 -32 8 1 -32 9 1 -32 10 1 -32 11 1 -32 12 1 -32 13 1 -32 14 1 -32 15 1 -32 16 1 -32 17 1 -32 18 1 -32 19 1 -32 20 1 -32 21 1 -32 22 1 -32 23 1 -32 24 1 -32 25 1 -32 26 1 -32 27 1 -32 28 1 -32 29 1 -32 30 1 -32 31 1 -32 32 1 -32 33 1 -32 34 1 -32 35 1 -32 36 1 -32 37 1 -32 38 1 -32 39 1 -32 40 1 -32 41 1 -32 42 1 -32 43 1 -32 44 1 -32 45 1 -32 46 1 -32 47 1 -32 48 1 -32 49 1 -32 50 1 -32 51 1 -32 52 1 -32 53 1 -32 54 1 -32 55 1 -32 56 1 -32 57 1 -32 58 1 -32 59 1 -32 60 1 -32 61 1 -32 62 1 -32 63 1 -32 64 1 -32 65 1 -32 66 1 -32 67 1 -32 68 1 -32 69 1 -32 70 1 -32 71 1 -32 72 1 -32 73 1 -32 74 1 -32 75 1 -32 76 1 -32 77 1 -32 78 1 -32 79 1 -32 80 1 -32 81 1 -32 82 1 -32 83 1 -32 84 1 -32 85 1 -32 86 1 -32 87 1 -32 88 1 -32 89 1 -32 90 1 -32 91 1 -32 92 1 -32 93 1 -32 94 1 -32 95 1 -32 96 1 -32 97 1 -32 98 1 -32 99 1 -32 100 1 -33 0 1 -33 1 1 -33 2 1 -33 3 1 -33 4 1 -33 5 1 -33 6 1 -33 7 1 -33 8 1 -33 9 1 -33 10 1 -33 11 1 -33 12 1 -33 13 1 -33 14 1 -33 15 1 -33 16 1 -33 17 1 -33 18 1 -33 19 1 -33 20 1 -33 21 1 -33 22 1 -33 23 1 -33 24 1 -33 25 1 -33 26 1 -33 27 1 -33 28 1 -33 29 1 -33 30 1 -33 31 1 -33 32 1 -33 33 1 -33 34 1 -33 35 1 -33 36 1 -33 37 1 -33 38 1 -33 39 1 -33 40 1 -33 41 1 -33 42 1 -33 43 1 -33 44 1 -33 45 1 -33 46 1 -33 47 1 -33 48 1 -33 49 1 -33 50 1 -33 51 1 -33 52 1 -33 53 1 -33 54 1 -33 55 1 -33 56 1 -33 57 1 -33 58 1 -33 59 1 -33 60 1 -33 61 1 -33 62 1 -33 63 1 -33 64 1 -33 65 1 -33 66 1 -33 67 1 -33 68 1 -33 69 1 -33 70 1 -33 71 1 -33 72 1 -33 73 1 -33 74 1 -33 75 1 -33 76 1 -33 77 1 -33 78 1 -33 79 1 -33 80 1 -33 81 1 -33 82 1 -33 83 1 -33 84 1 -33 85 1 -33 86 1 -33 87 1 -33 88 1 -33 89 1 -33 90 1 -33 91 1 -33 92 1 -33 93 1 -33 94 1 -33 95 1 -33 96 1 -33 97 1 -33 98 1 -33 99 1 -33 100 1 -34 0 1 -34 1 1 -34 2 1 -34 3 1 -34 4 1 -34 5 1 -34 6 1 -34 7 1 -34 8 1 -34 9 1 -34 10 1 -34 11 1 -34 12 1 -34 13 1 -34 14 1 -34 15 1 -34 16 1 -34 17 1 -34 18 1 -34 19 1 -34 20 1 -34 21 1 -34 22 1 -34 23 1 -34 24 1 -34 25 1 -34 26 1 -34 27 1 -34 28 1 -34 29 1 -34 30 1 -34 31 1 -34 32 1 -34 33 1 -34 34 1 -34 35 1 -34 36 1 -34 37 1 -34 38 1 -34 39 1 -34 40 1 -34 41 1 -34 42 1 -34 43 1 -34 44 1 -34 45 1 -34 46 1 -34 47 1 -34 48 1 -34 49 1 -34 50 1 -34 51 1 -34 52 1 -34 53 1 -34 54 1 -34 55 1 -34 56 1 -34 57 1 -34 58 1 -34 59 1 -34 60 1 -34 61 1 -34 62 1 -34 63 1 -34 64 1 -34 65 1 -34 66 1 -34 67 1 -34 68 1 -34 69 1 -34 70 1 -34 71 1 -34 72 1 -34 73 1 -34 74 1 -34 75 1 -34 76 1 -34 77 1 -34 78 1 -34 79 1 -34 80 1 -34 81 1 -34 82 1 -34 83 1 -34 84 1 -34 85 1 -34 86 1 -34 87 1 -34 88 1 -34 89 1 -34 90 1 -34 91 1 -34 92 1 -34 93 1 -34 94 1 -34 95 1 -34 96 1 -34 97 1 -34 98 1 -34 99 1 -34 100 1 -35 0 1 -35 1 1 -35 2 1 -35 3 1 -35 4 1 -35 5 1 -35 6 1 -35 7 1 -35 8 1 -35 9 1 -35 10 1 -35 11 1 -35 12 1 -35 13 1 -35 14 1 -35 15 1 -35 16 1 -35 17 1 -35 18 1 -35 19 1 -35 20 1 -35 21 1 -35 22 1 -35 23 1 -35 24 1 -35 25 1 -35 26 1 -35 27 1 -35 28 1 -35 29 1 -35 30 1 -35 31 1 -35 32 1 -35 33 1 -35 34 1 -35 35 1 -35 36 1 -35 37 1 -35 38 1 -35 39 1 -35 40 1 -35 41 1 -35 42 1 -35 43 1 -35 44 1 -35 45 1 -35 46 1 -35 47 1 -35 48 1 -35 49 1 -35 50 1 -35 51 1 -35 52 1 -35 53 1 -35 54 1 -35 55 1 -35 56 1 -35 57 1 -35 58 1 -35 59 1 -35 60 1 -35 61 1 -35 62 1 -35 63 1 -35 64 1 -35 65 1 -35 66 1 -35 67 1 -35 68 1 -35 69 1 -35 70 1 -35 71 1 -35 72 1 -35 73 1 -35 74 1 -35 75 1 -35 76 1 -35 77 1 -35 78 1 -35 79 1 -35 80 1 -35 81 1 -35 82 1 -35 83 1 -35 84 1 -35 85 1 -35 86 1 -35 87 1 -35 88 1 -35 89 1 -35 90 1 -35 91 1 -35 92 1 -35 93 1 -35 94 1 -35 95 1 -35 96 1 -35 97 1 -35 98 1 -35 99 1 -35 100 1 -36 0 1 -36 1 1 -36 2 1 -36 3 1 -36 4 1 -36 5 1 -36 6 1 -36 7 1 -36 8 1 -36 9 1 -36 10 1 -36 11 1 -36 12 1 -36 13 1 -36 14 1 -36 15 1 -36 16 1 -36 17 1 -36 18 1 -36 19 1 -36 20 1 -36 21 1 -36 22 1 -36 23 1 -36 24 1 -36 25 1 -36 26 1 -36 27 1 -36 28 1 -36 29 1 -36 30 1 -36 31 1 -36 32 1 -36 33 1 -36 34 1 -36 35 1 -36 36 1 -36 37 1 -36 38 1 -36 39 1 -36 40 1 -36 41 1 -36 42 1 -36 43 1 -36 44 1 -36 45 1 -36 46 1 -36 47 1 -36 48 1 -36 49 1 -36 50 1 -36 51 1 -36 52 1 -36 53 1 -36 54 1 -36 55 1 -36 56 1 -36 57 1 -36 58 1 -36 59 1 -36 60 1 -36 61 1 -36 62 1 -36 63 1 -36 64 1 -36 65 1 -36 66 1 -36 67 1 -36 68 1 -36 69 1 -36 70 1 -36 71 1 -36 72 1 -36 73 1 -36 74 1 -36 75 1 -36 76 1 -36 77 1 -36 78 1 -36 79 1 -36 80 1 -36 81 1 -36 82 1 -36 83 1 -36 84 1 -36 85 1 -36 86 1 -36 87 1 -36 88 1 -36 89 1 -36 90 1 -36 91 1 -36 92 1 -36 93 1 -36 94 1 -36 95 1 -36 96 1 -36 97 1 -36 98 1 -36 99 1 -36 100 1 -37 0 1 -37 1 1 -37 2 1 -37 3 1 -37 4 1 -37 5 1 -37 6 1 -37 7 1 -37 8 1 -37 9 1 -37 10 1 -37 11 1 -37 12 1 -37 13 1 -37 14 1 -37 15 1 -37 16 1 -37 17 1 -37 18 1 -37 19 1 -37 20 1 -37 21 1 -37 22 1 -37 23 1 -37 24 1 -37 25 1 -37 26 1 -37 27 1 -37 28 1 -37 29 1 -37 30 1 -37 31 1 -37 32 1 -37 33 1 -37 34 1 -37 35 1 -37 36 1 -37 37 1 -37 38 1 -37 39 1 -37 40 1 -37 41 1 -37 42 1 -37 43 1 -37 44 1 -37 45 1 -37 46 1 -37 47 1 -37 48 1 -37 49 1 -37 50 1 -37 51 1 -37 52 1 -37 53 1 -37 54 1 -37 55 1 -37 56 1 -37 57 1 -37 58 1 -37 59 1 -37 60 1 -37 61 1 -37 62 1 -37 63 1 -37 64 1 -37 65 1 -37 66 1 -37 67 1 -37 68 1 -37 69 1 -37 70 1 -37 71 1 -37 72 1 -37 73 1 -37 74 1 -37 75 1 -37 76 1 -37 77 1 -37 78 1 -37 79 1 -37 80 1 -37 81 1 -37 82 1 -37 83 1 -37 84 1 -37 85 1 -37 86 1 -37 87 1 -37 88 1 -37 89 1 -37 90 1 -37 91 1 -37 92 1 -37 93 1 -37 94 1 -37 95 1 -37 96 1 -37 97 1 -37 98 1 -37 99 1 -37 100 1 -38 0 1 -38 1 1 -38 2 1 -38 3 1 -38 4 1 -38 5 1 -38 6 1 -38 7 1 -38 8 1 -38 9 1 -38 10 1 -38 11 1 -38 12 1 -38 13 1 -38 14 1 -38 15 1 -38 16 1 -38 17 1 -38 18 1 -38 19 1 -38 20 1 -38 21 1 -38 22 1 -38 23 1 -38 24 1 -38 25 1 -38 26 1 -38 27 1 -38 28 1 -38 29 1 -38 30 1 -38 31 1 -38 32 1 -38 33 1 -38 34 1 -38 35 1 -38 36 1 -38 37 1 -38 38 1 -38 39 1 -38 40 1 -38 41 1 -38 42 1 -38 43 1 -38 44 1 -38 45 1 -38 46 1 -38 47 1 -38 48 1 -38 49 1 -38 50 1 -38 51 1 -38 52 1 -38 53 1 -38 54 1 -38 55 1 -38 56 1 -38 57 1 -38 58 1 -38 59 1 -38 60 1 -38 61 1 -38 62 1 -38 63 1 -38 64 1 -38 65 1 -38 66 1 -38 67 1 -38 68 1 -38 69 1 -38 70 1 -38 71 1 -38 72 1 -38 73 1 -38 74 1 -38 75 1 -38 76 1 -38 77 1 -38 78 1 -38 79 1 -38 80 1 -38 81 1 -38 82 1 -38 83 1 -38 84 1 -38 85 1 -38 86 1 -38 87 1 -38 88 1 -38 89 1 -38 90 1 -38 91 1 -38 92 1 -38 93 1 -38 94 1 -38 95 1 -38 96 1 -38 97 1 -38 98 1 -38 99 1 -38 100 1 -39 0 1 -39 1 1 -39 2 1 -39 3 1 -39 4 1 -39 5 1 -39 6 1 -39 7 1 -39 8 1 -39 9 1 -39 10 1 -39 11 1 -39 12 1 -39 13 1 -39 14 1 -39 15 1 -39 16 1 -39 17 1 -39 18 1 -39 19 1 -39 20 1 -39 21 1 -39 22 1 -39 23 1 -39 24 1 -39 25 1 -39 26 1 -39 27 1 -39 28 1 -39 29 1 -39 30 1 -39 31 1 -39 32 1 -39 33 1 -39 34 1 -39 35 1 -39 36 1 -39 37 1 -39 38 1 -39 39 1 -39 40 1 -39 41 1 -39 42 1 -39 43 1 -39 44 1 -39 45 1 -39 46 1 -39 47 1 -39 48 1 -39 49 1 -39 50 1 -39 51 1 -39 52 1 -39 53 1 -39 54 1 -39 55 1 -39 56 1 -39 57 1 -39 58 1 -39 59 1 -39 60 1 -39 61 1 -39 62 1 -39 63 1 -39 64 1 -39 65 1 -39 66 1 -39 67 1 -39 68 1 -39 69 1 -39 70 1 -39 71 1 -39 72 1 -39 73 1 -39 74 1 -39 75 1 -39 76 1 -39 77 1 -39 78 1 -39 79 1 -39 80 1 -39 81 1 -39 82 1 -39 83 1 -39 84 1 -39 85 1 -39 86 1 -39 87 1 -39 88 1 -39 89 1 -39 90 1 -39 91 1 -39 92 1 -39 93 1 -39 94 1 -39 95 1 -39 96 1 -39 97 1 -39 98 1 -39 99 1 -39 100 1 -40 0 1 -40 1 1 -40 2 1 -40 3 1 -40 4 1 -40 5 1 -40 6 1 -40 7 1 -40 8 1 -40 9 1 -40 10 1 -40 11 1 -40 12 1 -40 13 1 -40 14 1 -40 15 1 -40 16 1 -40 17 1 -40 18 1 -40 19 1 -40 20 1 -40 21 1 -40 22 1 -40 23 1 -40 24 1 -40 25 1 -40 26 1 -40 27 1 -40 28 1 -40 29 1 -40 30 1 -40 31 1 -40 32 1 -40 33 1 -40 34 1 -40 35 1 -40 36 1 -40 37 1 -40 38 1 -40 39 1 -40 40 1 -40 41 1 -40 42 1 -40 43 1 -40 44 1 -40 45 1 -40 46 1 -40 47 1 -40 48 1 -40 49 1 -40 50 1 -40 51 1 -40 52 1 -40 53 1 -40 54 1 -40 55 1 -40 56 1 -40 57 1 -40 58 1 -40 59 1 -40 60 1 -40 61 1 -40 62 1 -40 63 1 -40 64 1 -40 65 1 -40 66 1 -40 67 1 -40 68 1 -40 69 1 -40 70 1 -40 71 1 -40 72 1 -40 73 1 -40 74 1 -40 75 1 -40 76 1 -40 77 1 -40 78 1 -40 79 1 -40 80 1 -40 81 1 -40 82 1 -40 83 1 -40 84 1 -40 85 1 -40 86 1 -40 87 1 -40 88 1 -40 89 1 -40 90 1 -40 91 1 -40 92 1 -40 93 1 -40 94 1 -40 95 1 -40 96 1 -40 97 1 -40 98 1 -40 99 1 -40 100 1 -41 0 1 -41 1 1 -41 2 1 -41 3 1 -41 4 1 -41 5 1 -41 6 1 -41 7 1 -41 8 1 -41 9 1 -41 10 1 -41 11 1 -41 12 1 -41 13 1 -41 14 1 -41 15 1 -41 16 1 -41 17 1 -41 18 1 -41 19 1 -41 20 1 -41 21 1 -41 22 1 -41 23 1 -41 24 1 -41 25 1 -41 26 1 -41 27 1 -41 28 1 -41 29 1 -41 30 1 -41 31 1 -41 32 1 -41 33 1 -41 34 1 -41 35 1 -41 36 1 -41 37 1 -41 38 1 -41 39 1 -41 40 1 -41 41 1 -41 42 1 -41 43 1 -41 44 1 -41 45 1 -41 46 1 -41 47 1 -41 48 1 -41 49 1 -41 50 1 -41 51 1 -41 52 1 -41 53 1 -41 54 1 -41 55 1 -41 56 1 -41 57 1 -41 58 1 -41 59 1 -41 60 1 -41 61 1 -41 62 1 -41 63 1 -41 64 1 -41 65 1 -41 66 1 -41 67 1 -41 68 1 -41 69 1 -41 70 1 -41 71 1 -41 72 1 -41 73 1 -41 74 1 -41 75 1 -41 76 1 -41 77 1 -41 78 1 -41 79 1 -41 80 1 -41 81 1 -41 82 1 -41 83 1 -41 84 1 -41 85 1 -41 86 1 -41 87 1 -41 88 1 -41 89 1 -41 90 1 -41 91 1 -41 92 1 -41 93 1 -41 94 1 -41 95 1 -41 96 1 -41 97 1 -41 98 1 -41 99 1 -41 100 1 -42 0 1 -42 1 1 -42 2 1 -42 3 1 -42 4 1 -42 5 1 -42 6 1 -42 7 1 -42 8 1 -42 9 1 -42 10 1 -42 11 1 -42 12 1 -42 13 1 -42 14 1 -42 15 1 -42 16 1 -42 17 1 -42 18 1 -42 19 1 -42 20 1 -42 21 1 -42 22 1 -42 23 1 -42 24 1 -42 25 1 -42 26 1 -42 27 1 -42 28 1 -42 29 1 -42 30 1 -42 31 1 -42 32 1 -42 33 1 -42 34 1 -42 35 1 -42 36 1 -42 37 1 -42 38 1 -42 39 1 -42 40 1 -42 41 1 -42 42 1 -42 43 1 -42 44 1 -42 45 1 -42 46 1 -42 47 1 -42 48 1 -42 49 1 -42 50 1 -42 51 1 -42 52 1 -42 53 1 -42 54 1 -42 55 1 -42 56 1 -42 57 1 -42 58 1 -42 59 1 -42 60 1 -42 61 1 -42 62 1 -42 63 1 -42 64 1 -42 65 1 -42 66 1 -42 67 1 -42 68 1 -42 69 1 -42 70 1 -42 71 1 -42 72 1 -42 73 1 -42 74 1 -42 75 1 -42 76 1 -42 77 1 -42 78 1 -42 79 1 -42 80 1 -42 81 1 -42 82 1 -42 83 1 -42 84 1 -42 85 1 -42 86 1 -42 87 1 -42 88 1 -42 89 1 -42 90 1 -42 91 1 -42 92 1 -42 93 1 -42 94 1 -42 95 1 -42 96 1 -42 97 1 -42 98 1 -42 99 1 -42 100 1 -43 0 1 -43 1 1 -43 2 1 -43 3 1 -43 4 1 -43 5 1 -43 6 1 -43 7 1 -43 8 1 -43 9 1 -43 10 1 -43 11 1 -43 12 1 -43 13 1 -43 14 1 -43 15 1 -43 16 1 -43 17 1 -43 18 1 -43 19 1 -43 20 1 -43 21 1 -43 22 1 -43 23 1 -43 24 1 -43 25 1 -43 26 1 -43 27 1 -43 28 1 -43 29 1 -43 30 1 -43 31 1 -43 32 1 -43 33 1 -43 34 1 -43 35 1 -43 36 1 -43 37 1 -43 38 1 -43 39 1 -43 40 1 -43 41 1 -43 42 1 -43 43 1 -43 44 1 -43 45 1 -43 46 1 -43 47 1 -43 48 1 -43 49 1 -43 50 1 -43 51 1 -43 52 1 -43 53 1 -43 54 1 -43 55 1 -43 56 1 -43 57 1 -43 58 1 -43 59 1 -43 60 1 -43 61 1 -43 62 1 -43 63 1 -43 64 1 -43 65 1 -43 66 1 -43 67 1 -43 68 1 -43 69 1 -43 70 1 -43 71 1 -43 72 1 -43 73 1 -43 74 1 -43 75 1 -43 76 1 -43 77 1 -43 78 1 -43 79 1 -43 80 1 -43 81 1 -43 82 1 -43 83 1 -43 84 1 -43 85 1 -43 86 1 -43 87 1 -43 88 1 -43 89 1 -43 90 1 -43 91 1 -43 92 1 -43 93 1 -43 94 1 -43 95 1 -43 96 1 -43 97 1 -43 98 1 -43 99 1 -43 100 1 -44 0 1 -44 1 1 -44 2 1 -44 3 1 -44 4 1 -44 5 1 -44 6 1 -44 7 1 -44 8 1 -44 9 1 -44 10 1 -44 11 1 -44 12 1 -44 13 1 -44 14 1 -44 15 1 -44 16 1 -44 17 1 -44 18 1 -44 19 1 -44 20 1 -44 21 1 -44 22 1 -44 23 1 -44 24 1 -44 25 1 -44 26 1 -44 27 1 -44 28 1 -44 29 1 -44 30 1 -44 31 1 -44 32 1 -44 33 1 -44 34 1 -44 35 1 -44 36 1 -44 37 1 -44 38 1 -44 39 1 -44 40 1 -44 41 1 -44 42 1 -44 43 1 -44 44 1 -44 45 1 -44 46 1 -44 47 1 -44 48 1 -44 49 1 -44 50 1 -44 51 1 -44 52 1 -44 53 1 -44 54 1 -44 55 1 -44 56 1 -44 57 1 -44 58 1 -44 59 1 -44 60 1 -44 61 1 -44 62 1 -44 63 1 -44 64 1 -44 65 1 -44 66 1 -44 67 1 -44 68 1 -44 69 1 -44 70 1 -44 71 1 -44 72 1 -44 73 1 -44 74 1 -44 75 1 -44 76 1 -44 77 1 -44 78 1 -44 79 1 -44 80 1 -44 81 1 -44 82 1 -44 83 1 -44 84 1 -44 85 1 -44 86 1 -44 87 1 -44 88 1 -44 89 1 -44 90 1 -44 91 1 -44 92 1 -44 93 1 -44 94 1 -44 95 1 -44 96 1 -44 97 1 -44 98 1 -44 99 1 -44 100 1 -45 0 1 -45 1 1 -45 2 1 -45 3 1 -45 4 1 -45 5 1 -45 6 1 -45 7 1 -45 8 1 -45 9 1 -45 10 1 -45 11 1 -45 12 1 -45 13 1 -45 14 1 -45 15 1 -45 16 1 -45 17 1 -45 18 1 -45 19 1 -45 20 1 -45 21 1 -45 22 1 -45 23 1 -45 24 1 -45 25 1 -45 26 1 -45 27 1 -45 28 1 -45 29 1 -45 30 1 -45 31 1 -45 32 1 -45 33 1 -45 34 1 -45 35 1 -45 36 1 -45 37 1 -45 38 1 -45 39 1 -45 40 1 -45 41 1 -45 42 1 -45 43 1 -45 44 1 -45 45 1 -45 46 1 -45 47 1 -45 48 1 -45 49 1 -45 50 1 -45 51 1 -45 52 1 -45 53 1 -45 54 1 -45 55 1 -45 56 1 -45 57 1 -45 58 1 -45 59 1 -45 60 1 -45 61 1 -45 62 1 -45 63 1 -45 64 1 -45 65 1 -45 66 1 -45 67 1 -45 68 1 -45 69 1 -45 70 1 -45 71 1 -45 72 1 -45 73 1 -45 74 1 -45 75 1 -45 76 1 -45 77 1 -45 78 1 -45 79 1 -45 80 1 -45 81 1 -45 82 1 -45 83 1 -45 84 1 -45 85 1 -45 86 1 -45 87 1 -45 88 1 -45 89 1 -45 90 1 -45 91 1 -45 92 1 -45 93 1 -45 94 1 -45 95 1 -45 96 1 -45 97 1 -45 98 1 -45 99 1 -45 100 1 -46 0 1 -46 1 1 -46 2 1 -46 3 1 -46 4 1 -46 5 1 -46 6 1 -46 7 1 -46 8 1 -46 9 1 -46 10 1 -46 11 1 -46 12 1 -46 13 1 -46 14 1 -46 15 1 -46 16 1 -46 17 1 -46 18 1 -46 19 1 -46 20 1 -46 21 1 -46 22 1 -46 23 1 -46 24 1 -46 25 1 -46 26 1 -46 27 1 -46 28 1 -46 29 1 -46 30 1 -46 31 1 -46 32 1 -46 33 1 -46 34 1 -46 35 1 -46 36 1 -46 37 1 -46 38 1 -46 39 1 -46 40 1 -46 41 1 -46 42 1 -46 43 1 -46 44 1 -46 45 1 -46 46 1 -46 47 1 -46 48 1 -46 49 1 -46 50 1 -46 51 1 -46 52 1 -46 53 1 -46 54 1 -46 55 1 -46 56 1 -46 57 1 -46 58 1 -46 59 1 -46 60 1 -46 61 1 -46 62 1 -46 63 1 -46 64 1 -46 65 1 -46 66 1 -46 67 1 -46 68 1 -46 69 1 -46 70 1 -46 71 1 -46 72 1 -46 73 1 -46 74 1 -46 75 1 -46 76 1 -46 77 1 -46 78 1 -46 79 1 -46 80 1 -46 81 1 -46 82 1 -46 83 1 -46 84 1 -46 85 1 -46 86 1 -46 87 1 -46 88 1 -46 89 1 -46 90 1 -46 91 1 -46 92 1 -46 93 1 -46 94 1 -46 95 1 -46 96 1 -46 97 1 -46 98 1 -46 99 1 -46 100 1 -47 0 1 -47 1 1 -47 2 1 -47 3 1 -47 4 1 -47 5 1 -47 6 1 -47 7 1 -47 8 1 -47 9 1 -47 10 1 -47 11 1 -47 12 1 -47 13 1 -47 14 1 -47 15 1 -47 16 1 -47 17 1 -47 18 1 -47 19 1 -47 20 1 -47 21 1 -47 22 1 -47 23 1 -47 24 1 -47 25 1 -47 26 1 -47 27 1 -47 28 1 -47 29 1 -47 30 1 -47 31 1 -47 32 1 -47 33 1 -47 34 1 -47 35 1 -47 36 1 -47 37 1 -47 38 1 -47 39 1 -47 40 1 -47 41 1 -47 42 1 -47 43 1 -47 44 1 -47 45 1 -47 46 1 -47 47 1 -47 48 1 -47 49 1 -47 50 1 -47 51 1 -47 52 1 -47 53 1 -47 54 1 -47 55 1 -47 56 1 -47 57 1 -47 58 1 -47 59 1 -47 60 1 -47 61 1 -47 62 1 -47 63 1 -47 64 1 -47 65 1 -47 66 1 -47 67 1 -47 68 1 -47 69 1 -47 70 1 -47 71 1 -47 72 1 -47 73 1 -47 74 1 -47 75 1 -47 76 1 -47 77 1 -47 78 1 -47 79 1 -47 80 1 -47 81 1 -47 82 1 -47 83 1 -47 84 1 -47 85 1 -47 86 1 -47 87 1 -47 88 1 -47 89 1 -47 90 1 -47 91 1 -47 92 1 -47 93 1 -47 94 1 -47 95 1 -47 96 1 -47 97 1 -47 98 1 -47 99 1 -47 100 1 -48 0 1 -48 1 1 -48 2 1 -48 3 1 -48 4 1 -48 5 1 -48 6 1 -48 7 1 -48 8 1 -48 9 1 -48 10 1 -48 11 1 -48 12 1 -48 13 1 -48 14 1 -48 15 1 -48 16 1 -48 17 1 -48 18 1 -48 19 1 -48 20 1 -48 21 1 -48 22 1 -48 23 1 -48 24 1 -48 25 1 -48 26 1 -48 27 1 -48 28 1 -48 29 1 -48 30 1 -48 31 1 -48 32 1 -48 33 1 -48 34 1 -48 35 1 -48 36 1 -48 37 1 -48 38 1 -48 39 1 -48 40 1 -48 41 1 -48 42 1 -48 43 1 -48 44 1 -48 45 1 -48 46 1 -48 47 1 -48 48 1 -48 49 1 -48 50 1 -48 51 1 -48 52 1 -48 53 1 -48 54 1 -48 55 1 -48 56 1 -48 57 1 -48 58 1 -48 59 1 -48 60 1 -48 61 1 -48 62 1 -48 63 1 -48 64 1 -48 65 1 -48 66 1 -48 67 1 -48 68 1 -48 69 1 -48 70 1 -48 71 1 -48 72 1 -48 73 1 -48 74 1 -48 75 1 -48 76 1 -48 77 1 -48 78 1 -48 79 1 -48 80 1 -48 81 1 -48 82 1 -48 83 1 -48 84 1 -48 85 1 -48 86 1 -48 87 1 -48 88 1 -48 89 1 -48 90 1 -48 91 1 -48 92 1 -48 93 1 -48 94 1 -48 95 1 -48 96 1 -48 97 1 -48 98 1 -48 99 1 -48 100 1 -49 0 1 -49 1 1 -49 2 1 -49 3 1 -49 4 1 -49 5 1 -49 6 1 -49 7 1 -49 8 1 -49 9 1 -49 10 1 -49 11 1 -49 12 1 -49 13 1 -49 14 1 -49 15 1 -49 16 1 -49 17 1 -49 18 1 -49 19 1 -49 20 1 -49 21 1 -49 22 1 -49 23 1 -49 24 1 -49 25 1 -49 26 1 -49 27 1 -49 28 1 -49 29 1 -49 30 1 -49 31 1 -49 32 1 -49 33 1 -49 34 1 -49 35 1 -49 36 1 -49 37 1 -49 38 1 -49 39 1 -49 40 1 -49 41 1 -49 42 1 -49 43 1 -49 44 1 -49 45 1 -49 46 1 -49 47 1 -49 48 1 -49 49 1 -49 50 1 -49 51 1 -49 52 1 -49 53 1 -49 54 1 -49 55 1 -49 56 1 -49 57 1 -49 58 1 -49 59 1 -49 60 1 -49 61 1 -49 62 1 -49 63 1 -49 64 1 -49 65 1 -49 66 1 -49 67 1 -49 68 1 -49 69 1 -49 70 1 -49 71 1 -49 72 1 -49 73 1 -49 74 1 -49 75 1 -49 76 1 -49 77 1 -49 78 1 -49 79 1 -49 80 1 -49 81 1 -49 82 1 -49 83 1 -49 84 1 -49 85 1 -49 86 1 -49 87 1 -49 88 1 -49 89 1 -49 90 1 -49 91 1 -49 92 1 -49 93 1 -49 94 1 -49 95 1 -49 96 1 -49 97 1 -49 98 1 -49 99 1 -49 100 1 -50 0 1 -50 1 1 -50 2 1 -50 3 1 -50 4 1 -50 5 1 -50 6 1 -50 7 1 -50 8 1 -50 9 1 -50 10 1 -50 11 1 -50 12 1 -50 13 1 -50 14 1 -50 15 1 -50 16 1 -50 17 1 -50 18 1 -50 19 1 -50 20 1 -50 21 1 -50 22 1 -50 23 1 -50 24 1 -50 25 1 -50 26 1 -50 27 1 -50 28 1 -50 29 1 -50 30 1 -50 31 1 -50 32 1 -50 33 1 -50 34 1 -50 35 1 -50 36 1 -50 37 1 -50 38 1 -50 39 1 -50 40 1 -50 41 1 -50 42 1 -50 43 1 -50 44 1 -50 45 1 -50 46 1 -50 47 1 -50 48 1 -50 49 1 -50 50 1 -50 51 1 -50 52 1 -50 53 1 -50 54 1 -50 55 1 -50 56 1 -50 57 1 -50 58 1 -50 59 1 -50 60 1 -50 61 1 -50 62 1 -50 63 1 -50 64 1 -50 65 1 -50 66 1 -50 67 1 -50 68 1 -50 69 1 -50 70 1 -50 71 1 -50 72 1 -50 73 1 -50 74 1 -50 75 1 -50 76 1 -50 77 1 -50 78 1 -50 79 1 -50 80 1 -50 81 1 -50 82 1 -50 83 1 -50 84 1 -50 85 1 -50 86 1 -50 87 1 -50 88 1 -50 89 1 -50 90 1 -50 91 1 -50 92 1 -50 93 1 -50 94 1 -50 95 1 -50 96 1 -50 97 1 -50 98 1 -50 99 1 -50 100 1 -51 0 1 -51 1 1 -51 2 1 -51 3 1 -51 4 1 -51 5 1 -51 6 1 -51 7 1 -51 8 1 -51 9 1 -51 10 1 -51 11 1 -51 12 1 -51 13 1 -51 14 1 -51 15 1 -51 16 1 -51 17 1 -51 18 1 -51 19 1 -51 20 1 -51 21 1 -51 22 1 -51 23 1 -51 24 1 -51 25 1 -51 26 1 -51 27 1 -51 28 1 -51 29 1 -51 30 1 -51 31 1 -51 32 1 -51 33 1 -51 34 1 -51 35 1 -51 36 1 -51 37 1 -51 38 1 -51 39 1 -51 40 1 -51 41 1 -51 42 1 -51 43 1 -51 44 1 -51 45 1 -51 46 1 -51 47 1 -51 48 1 -51 49 1 -51 50 1 -51 51 1 -51 52 1 -51 53 1 -51 54 1 -51 55 1 -51 56 1 -51 57 1 -51 58 1 -51 59 1 -51 60 1 -51 61 1 -51 62 1 -51 63 1 -51 64 1 -51 65 1 -51 66 1 -51 67 1 -51 68 1 -51 69 1 -51 70 1 -51 71 1 -51 72 1 -51 73 1 -51 74 1 -51 75 1 -51 76 1 -51 77 1 -51 78 1 -51 79 1 -51 80 1 -51 81 1 -51 82 1 -51 83 1 -51 84 1 -51 85 1 -51 86 1 -51 87 1 -51 88 1 -51 89 1 -51 90 1 -51 91 1 -51 92 1 -51 93 1 -51 94 1 -51 95 1 -51 96 1 -51 97 1 -51 98 1 -51 99 1 -51 100 1 -52 0 1 -52 1 1 -52 2 1 -52 3 1 -52 4 1 -52 5 1 -52 6 1 -52 7 1 -52 8 1 -52 9 1 -52 10 1 -52 11 1 -52 12 1 -52 13 1 -52 14 1 -52 15 1 -52 16 1 -52 17 1 -52 18 1 -52 19 1 -52 20 1 -52 21 1 -52 22 1 -52 23 1 -52 24 1 -52 25 1 -52 26 1 -52 27 1 -52 28 1 -52 29 1 -52 30 1 -52 31 1 -52 32 1 -52 33 1 -52 34 1 -52 35 1 -52 36 1 -52 37 1 -52 38 1 -52 39 1 -52 40 1 -52 41 1 -52 42 1 -52 43 1 -52 44 1 -52 45 1 -52 46 1 -52 47 1 -52 48 1 -52 49 1 -52 50 1 -52 51 1 -52 52 1 -52 53 1 -52 54 1 -52 55 1 -52 56 1 -52 57 1 -52 58 1 -52 59 1 -52 60 1 -52 61 1 -52 62 1 -52 63 1 -52 64 1 -52 65 1 -52 66 1 -52 67 1 -52 68 1 -52 69 1 -52 70 1 -52 71 1 -52 72 1 -52 73 1 -52 74 1 -52 75 1 -52 76 1 -52 77 1 -52 78 1 -52 79 1 -52 80 1 -52 81 1 -52 82 1 -52 83 1 -52 84 1 -52 85 1 -52 86 1 -52 87 1 -52 88 1 -52 89 1 -52 90 1 -52 91 1 -52 92 1 -52 93 1 -52 94 1 -52 95 1 -52 96 1 -52 97 1 -52 98 1 -52 99 1 -52 100 1 -53 0 1 -53 1 1 -53 2 1 -53 3 1 -53 4 1 -53 5 1 -53 6 1 -53 7 1 -53 8 1 -53 9 1 -53 10 1 -53 11 1 -53 12 1 -53 13 1 -53 14 1 -53 15 1 -53 16 1 -53 17 1 -53 18 1 -53 19 1 -53 20 1 -53 21 1 -53 22 1 -53 23 1 -53 24 1 -53 25 1 -53 26 1 -53 27 1 -53 28 1 -53 29 1 -53 30 1 -53 31 1 -53 32 1 -53 33 1 -53 34 1 -53 35 1 -53 36 1 -53 37 1 -53 38 1 -53 39 1 -53 40 1 -53 41 1 -53 42 1 -53 43 1 -53 44 1 -53 45 1 -53 46 1 -53 47 1 -53 48 1 -53 49 1 -53 50 1 -53 51 1 -53 52 1 -53 53 1 -53 54 1 -53 55 1 -53 56 1 -53 57 1 -53 58 1 -53 59 1 -53 60 1 -53 61 1 -53 62 1 -53 63 1 -53 64 1 -53 65 1 -53 66 1 -53 67 1 -53 68 1 -53 69 1 -53 70 1 -53 71 1 -53 72 1 -53 73 1 -53 74 1 -53 75 1 -53 76 1 -53 77 1 -53 78 1 -53 79 1 -53 80 1 -53 81 1 -53 82 1 -53 83 1 -53 84 1 -53 85 1 -53 86 1 -53 87 1 -53 88 1 -53 89 1 -53 90 1 -53 91 1 -53 92 1 -53 93 1 -53 94 1 -53 95 1 -53 96 1 -53 97 1 -53 98 1 -53 99 1 -53 100 1 -54 0 1 -54 1 1 -54 2 1 -54 3 1 -54 4 1 -54 5 1 -54 6 1 -54 7 1 -54 8 1 -54 9 1 -54 10 1 -54 11 1 -54 12 1 -54 13 1 -54 14 1 -54 15 1 -54 16 1 -54 17 1 -54 18 1 -54 19 1 -54 20 1 -54 21 1 -54 22 1 -54 23 1 -54 24 1 -54 25 1 -54 26 1 -54 27 1 -54 28 1 -54 29 1 -54 30 1 -54 31 1 -54 32 1 -54 33 1 -54 34 1 -54 35 1 -54 36 1 -54 37 1 -54 38 1 -54 39 1 -54 40 1 -54 41 1 -54 42 1 -54 43 1 -54 44 1 -54 45 1 -54 46 1 -54 47 1 -54 48 1 -54 49 1 -54 50 1 -54 51 1 -54 52 1 -54 53 1 -54 54 1 -54 55 1 -54 56 1 -54 57 1 -54 58 1 -54 59 1 -54 60 1 -54 61 1 -54 62 1 -54 63 1 -54 64 1 -54 65 1 -54 66 1 -54 67 1 -54 68 1 -54 69 1 -54 70 1 -54 71 1 -54 72 1 -54 73 1 -54 74 1 -54 75 1 -54 76 1 -54 77 1 -54 78 1 -54 79 1 -54 80 1 -54 81 1 -54 82 1 -54 83 1 -54 84 1 -54 85 1 -54 86 1 -54 87 1 -54 88 1 -54 89 1 -54 90 1 -54 91 1 -54 92 1 -54 93 1 -54 94 1 -54 95 1 -54 96 1 -54 97 1 -54 98 1 -54 99 1 -54 100 1 -55 0 1 -55 1 1 -55 2 1 -55 3 1 -55 4 1 -55 5 1 -55 6 1 -55 7 1 -55 8 1 -55 9 1 -55 10 1 -55 11 1 -55 12 1 -55 13 1 -55 14 1 -55 15 1 -55 16 1 -55 17 1 -55 18 1 -55 19 1 -55 20 1 -55 21 1 -55 22 1 -55 23 1 -55 24 1 -55 25 1 -55 26 1 -55 27 1 -55 28 1 -55 29 1 -55 30 1 -55 31 1 -55 32 1 -55 33 1 -55 34 1 -55 35 1 -55 36 1 -55 37 1 -55 38 1 -55 39 1 -55 40 1 -55 41 1 -55 42 1 -55 43 1 -55 44 1 -55 45 1 -55 46 1 -55 47 1 -55 48 1 -55 49 1 -55 50 1 -55 51 1 -55 52 1 -55 53 1 -55 54 1 -55 55 1 -55 56 1 -55 57 1 -55 58 1 -55 59 1 -55 60 1 -55 61 1 -55 62 1 -55 63 1 -55 64 1 -55 65 1 -55 66 1 -55 67 1 -55 68 1 -55 69 1 -55 70 1 -55 71 1 -55 72 1 -55 73 1 -55 74 1 -55 75 1 -55 76 1 -55 77 1 -55 78 1 -55 79 1 -55 80 1 -55 81 1 -55 82 1 -55 83 1 -55 84 1 -55 85 1 -55 86 1 -55 87 1 -55 88 1 -55 89 1 -55 90 1 -55 91 1 -55 92 1 -55 93 1 -55 94 1 -55 95 1 -55 96 1 -55 97 1 -55 98 1 -55 99 1 -55 100 1 -56 0 1 -56 1 1 -56 2 1 -56 3 1 -56 4 1 -56 5 1 -56 6 1 -56 7 1 -56 8 1 -56 9 1 -56 10 1 -56 11 1 -56 12 1 -56 13 1 -56 14 1 -56 15 1 -56 16 1 -56 17 1 -56 18 1 -56 19 1 -56 20 1 -56 21 1 -56 22 1 -56 23 1 -56 24 1 -56 25 1 -56 26 1 -56 27 1 -56 28 1 -56 29 1 -56 30 1 -56 31 1 -56 32 1 -56 33 1 -56 34 1 -56 35 1 -56 36 1 -56 37 1 -56 38 1 -56 39 1 -56 40 1 -56 41 1 -56 42 1 -56 43 1 -56 44 1 -56 45 1 -56 46 1 -56 47 1 -56 48 1 -56 49 1 -56 50 1 -56 51 1 -56 52 1 -56 53 1 -56 54 1 -56 55 1 -56 56 1 -56 57 1 -56 58 1 -56 59 1 -56 60 1 -56 61 1 -56 62 1 -56 63 1 -56 64 1 -56 65 1 -56 66 1 -56 67 1 -56 68 1 -56 69 1 -56 70 1 -56 71 1 -56 72 1 -56 73 1 -56 74 1 -56 75 1 -56 76 1 -56 77 1 -56 78 1 -56 79 1 -56 80 1 -56 81 1 -56 82 1 -56 83 1 -56 84 1 -56 85 1 -56 86 1 -56 87 1 -56 88 1 -56 89 1 -56 90 1 -56 91 1 -56 92 1 -56 93 1 -56 94 1 -56 95 1 -56 96 1 -56 97 1 -56 98 1 -56 99 1 -56 100 1 -57 0 1 -57 1 1 -57 2 1 -57 3 1 -57 4 1 -57 5 1 -57 6 1 -57 7 1 -57 8 1 -57 9 1 -57 10 1 -57 11 1 -57 12 1 -57 13 1 -57 14 1 -57 15 1 -57 16 1 -57 17 1 -57 18 1 -57 19 1 -57 20 1 -57 21 1 -57 22 1 -57 23 1 -57 24 1 -57 25 1 -57 26 1 -57 27 1 -57 28 1 -57 29 1 -57 30 1 -57 31 1 -57 32 1 -57 33 1 -57 34 1 -57 35 1 -57 36 1 -57 37 1 -57 38 1 -57 39 1 -57 40 1 -57 41 1 -57 42 1 -57 43 1 -57 44 1 -57 45 1 -57 46 1 -57 47 1 -57 48 1 -57 49 1 -57 50 1 -57 51 1 -57 52 1 -57 53 1 -57 54 1 -57 55 1 -57 56 1 -57 57 1 -57 58 1 -57 59 1 -57 60 1 -57 61 1 -57 62 1 -57 63 1 -57 64 1 -57 65 1 -57 66 1 -57 67 1 -57 68 1 -57 69 1 -57 70 1 -57 71 1 -57 72 1 -57 73 1 -57 74 1 -57 75 1 -57 76 1 -57 77 1 -57 78 1 -57 79 1 -57 80 1 -57 81 1 -57 82 1 -57 83 1 -57 84 1 -57 85 1 -57 86 1 -57 87 1 -57 88 1 -57 89 1 -57 90 1 -57 91 1 -57 92 1 -57 93 1 -57 94 1 -57 95 1 -57 96 1 -57 97 1 -57 98 1 -57 99 1 -57 100 1 -58 0 1 -58 1 1 -58 2 1 -58 3 1 -58 4 1 -58 5 1 -58 6 1 -58 7 1 -58 8 1 -58 9 1 -58 10 1 -58 11 1 -58 12 1 -58 13 1 -58 14 1 -58 15 1 -58 16 1 -58 17 1 -58 18 1 -58 19 1 -58 20 1 -58 21 1 -58 22 1 -58 23 1 -58 24 1 -58 25 1 -58 26 1 -58 27 1 -58 28 1 -58 29 1 -58 30 1 -58 31 1 -58 32 1 -58 33 1 -58 34 1 -58 35 1 -58 36 1 -58 37 1 -58 38 1 -58 39 1 -58 40 1 -58 41 1 -58 42 1 -58 43 1 -58 44 1 -58 45 1 -58 46 1 -58 47 1 -58 48 1 -58 49 1 -58 50 1 -58 51 1 -58 52 1 -58 53 1 -58 54 1 -58 55 1 -58 56 1 -58 57 1 -58 58 1 -58 59 1 -58 60 1 -58 61 1 -58 62 1 -58 63 1 -58 64 1 -58 65 1 -58 66 1 -58 67 1 -58 68 1 -58 69 1 -58 70 1 -58 71 1 -58 72 1 -58 73 1 -58 74 1 -58 75 1 -58 76 1 -58 77 1 -58 78 1 -58 79 1 -58 80 1 -58 81 1 -58 82 1 -58 83 1 -58 84 1 -58 85 1 -58 86 1 -58 87 1 -58 88 1 -58 89 1 -58 90 1 -58 91 1 -58 92 1 -58 93 1 -58 94 1 -58 95 1 -58 96 1 -58 97 1 -58 98 1 -58 99 1 -58 100 1 -59 0 1 -59 1 1 -59 2 1 -59 3 1 -59 4 1 -59 5 1 -59 6 1 -59 7 1 -59 8 1 -59 9 1 -59 10 1 -59 11 1 -59 12 1 -59 13 1 -59 14 1 -59 15 1 -59 16 1 -59 17 1 -59 18 1 -59 19 1 -59 20 1 -59 21 1 -59 22 1 -59 23 1 -59 24 1 -59 25 1 -59 26 1 -59 27 1 -59 28 1 -59 29 1 -59 30 1 -59 31 1 -59 32 1 -59 33 1 -59 34 1 -59 35 1 -59 36 1 -59 37 1 -59 38 1 -59 39 1 -59 40 1 -59 41 1 -59 42 1 -59 43 1 -59 44 1 -59 45 1 -59 46 1 -59 47 1 -59 48 1 -59 49 1 -59 50 1 -59 51 1 -59 52 1 -59 53 1 -59 54 1 -59 55 1 -59 56 1 -59 57 1 -59 58 1 -59 59 1 -59 60 1 -59 61 1 -59 62 1 -59 63 1 -59 64 1 -59 65 1 -59 66 1 -59 67 1 -59 68 1 -59 69 1 -59 70 1 -59 71 1 -59 72 1 -59 73 1 -59 74 1 -59 75 1 -59 76 1 -59 77 1 -59 78 1 -59 79 1 -59 80 1 -59 81 1 -59 82 1 -59 83 1 -59 84 1 -59 85 1 -59 86 1 -59 87 1 -59 88 1 -59 89 1 -59 90 1 -59 91 1 -59 92 1 -59 93 1 -59 94 1 -59 95 1 -59 96 1 -59 97 1 -59 98 1 -59 99 1 -59 100 1 -60 0 1 -60 1 1 -60 2 1 -60 3 1 -60 4 1 -60 5 1 -60 6 1 -60 7 1 -60 8 1 -60 9 1 -60 10 1 -60 11 1 -60 12 1 -60 13 1 -60 14 1 -60 15 1 -60 16 1 -60 17 1 -60 18 1 -60 19 1 -60 20 1 -60 21 1 -60 22 1 -60 23 1 -60 24 1 -60 25 1 -60 26 1 -60 27 1 -60 28 1 -60 29 1 -60 30 1 -60 31 1 -60 32 1 -60 33 1 -60 34 1 -60 35 1 -60 36 1 -60 37 1 -60 38 1 -60 39 1 -60 40 1 -60 41 1 -60 42 1 -60 43 1 -60 44 1 -60 45 1 -60 46 1 -60 47 1 -60 48 1 -60 49 1 -60 50 1 -60 51 1 -60 52 1 -60 53 1 -60 54 1 -60 55 1 -60 56 1 -60 57 1 -60 58 1 -60 59 1 -60 60 1 -60 61 1 -60 62 1 -60 63 1 -60 64 1 -60 65 1 -60 66 1 -60 67 1 -60 68 1 -60 69 1 -60 70 1 -60 71 1 -60 72 1 -60 73 1 -60 74 1 -60 75 1 -60 76 1 -60 77 1 -60 78 1 -60 79 1 -60 80 1 -60 81 1 -60 82 1 -60 83 1 -60 84 1 -60 85 1 -60 86 1 -60 87 1 -60 88 1 -60 89 1 -60 90 1 -60 91 1 -60 92 1 -60 93 1 -60 94 1 -60 95 1 -60 96 1 -60 97 1 -60 98 1 -60 99 1 -60 100 1 -61 0 1 -61 1 1 -61 2 1 -61 3 1 -61 4 1 -61 5 1 -61 6 1 -61 7 1 -61 8 1 -61 9 1 -61 10 1 -61 11 1 -61 12 1 -61 13 1 -61 14 1 -61 15 1 -61 16 1 -61 17 1 -61 18 1 -61 19 1 -61 20 1 -61 21 1 -61 22 1 -61 23 1 -61 24 1 -61 25 1 -61 26 1 -61 27 1 -61 28 1 -61 29 1 -61 30 1 -61 31 1 -61 32 1 -61 33 1 -61 34 1 -61 35 1 -61 36 1 -61 37 1 -61 38 1 -61 39 1 -61 40 1 -61 41 1 -61 42 1 -61 43 1 -61 44 1 -61 45 1 -61 46 1 -61 47 1 -61 48 1 -61 49 1 -61 50 1 -61 51 1 -61 52 1 -61 53 1 -61 54 1 -61 55 1 -61 56 1 -61 57 1 -61 58 1 -61 59 1 -61 60 1 -61 61 1 -61 62 1 -61 63 1 -61 64 1 -61 65 1 -61 66 1 -61 67 1 -61 68 1 -61 69 1 -61 70 1 -61 71 1 -61 72 1 -61 73 1 -61 74 1 -61 75 1 -61 76 1 -61 77 1 -61 78 1 -61 79 1 -61 80 1 -61 81 1 -61 82 1 -61 83 1 -61 84 1 -61 85 1 -61 86 1 -61 87 1 -61 88 1 -61 89 1 -61 90 1 -61 91 1 -61 92 1 -61 93 1 -61 94 1 -61 95 1 -61 96 1 -61 97 1 -61 98 1 -61 99 1 -61 100 1 -62 0 1 -62 1 1 -62 2 1 -62 3 1 -62 4 1 -62 5 1 -62 6 1 -62 7 1 -62 8 1 -62 9 1 -62 10 1 -62 11 1 -62 12 1 -62 13 1 -62 14 1 -62 15 1 -62 16 1 -62 17 1 -62 18 1 -62 19 1 -62 20 1 -62 21 1 -62 22 1 -62 23 1 -62 24 1 -62 25 1 -62 26 1 -62 27 1 -62 28 1 -62 29 1 -62 30 1 -62 31 1 -62 32 1 -62 33 1 -62 34 1 -62 35 1 -62 36 1 -62 37 1 -62 38 1 -62 39 1 -62 40 1 -62 41 1 -62 42 1 -62 43 1 -62 44 1 -62 45 1 -62 46 1 -62 47 1 -62 48 1 -62 49 1 -62 50 1 -62 51 1 -62 52 1 -62 53 1 -62 54 1 -62 55 1 -62 56 1 -62 57 1 -62 58 1 -62 59 1 -62 60 1 -62 61 1 -62 62 1 -62 63 1 -62 64 1 -62 65 1 -62 66 1 -62 67 1 -62 68 1 -62 69 1 -62 70 1 -62 71 1 -62 72 1 -62 73 1 -62 74 1 -62 75 1 -62 76 1 -62 77 1 -62 78 1 -62 79 1 -62 80 1 -62 81 1 -62 82 1 -62 83 1 -62 84 1 -62 85 1 -62 86 1 -62 87 1 -62 88 1 -62 89 1 -62 90 1 -62 91 1 -62 92 1 -62 93 1 -62 94 1 -62 95 1 -62 96 1 -62 97 1 -62 98 1 -62 99 1 -62 100 1 -63 0 1 -63 1 1 -63 2 1 -63 3 1 -63 4 1 -63 5 1 -63 6 1 -63 7 1 -63 8 1 -63 9 1 -63 10 1 -63 11 1 -63 12 1 -63 13 1 -63 14 1 -63 15 1 -63 16 1 -63 17 1 -63 18 1 -63 19 1 -63 20 1 -63 21 1 -63 22 1 -63 23 1 -63 24 1 -63 25 1 -63 26 1 -63 27 1 -63 28 1 -63 29 1 -63 30 1 -63 31 1 -63 32 1 -63 33 1 -63 34 1 -63 35 1 -63 36 1 -63 37 1 -63 38 1 -63 39 1 -63 40 1 -63 41 1 -63 42 1 -63 43 1 -63 44 1 -63 45 1 -63 46 1 -63 47 1 -63 48 1 -63 49 1 -63 50 1 -63 51 1 -63 52 1 -63 53 1 -63 54 1 -63 55 1 -63 56 1 -63 57 1 -63 58 1 -63 59 1 -63 60 1 -63 61 1 -63 62 1 -63 63 1 -63 64 1 -63 65 1 -63 66 1 -63 67 1 -63 68 1 -63 69 1 -63 70 1 -63 71 1 -63 72 1 -63 73 1 -63 74 1 -63 75 1 -63 76 1 -63 77 1 -63 78 1 -63 79 1 -63 80 1 -63 81 1 -63 82 1 -63 83 1 -63 84 1 -63 85 1 -63 86 1 -63 87 1 -63 88 1 -63 89 1 -63 90 1 -63 91 1 -63 92 1 -63 93 1 -63 94 1 -63 95 1 -63 96 1 -63 97 1 -63 98 1 -63 99 1 -63 100 1 -64 0 1 -64 1 1 -64 2 1 -64 3 1 -64 4 1 -64 5 1 -64 6 1 -64 7 1 -64 8 1 -64 9 1 -64 10 1 -64 11 1 -64 12 1 -64 13 1 -64 14 1 -64 15 1 -64 16 1 -64 17 1 -64 18 1 -64 19 1 -64 20 1 -64 21 1 -64 22 1 -64 23 1 -64 24 1 -64 25 1 -64 26 1 -64 27 1 -64 28 1 -64 29 1 -64 30 1 -64 31 1 -64 32 1 -64 33 1 -64 34 1 -64 35 1 -64 36 1 -64 37 1 -64 38 1 -64 39 1 -64 40 1 -64 41 1 -64 42 1 -64 43 1 -64 44 1 -64 45 1 -64 46 1 -64 47 1 -64 48 1 -64 49 1 -64 50 1 -64 51 1 -64 52 1 -64 53 1 -64 54 1 -64 55 1 -64 56 1 -64 57 1 -64 58 1 -64 59 1 -64 60 1 -64 61 1 -64 62 1 -64 63 1 -64 64 1 -64 65 1 -64 66 1 -64 67 1 -64 68 1 -64 69 1 -64 70 1 -64 71 1 -64 72 1 -64 73 1 -64 74 1 -64 75 1 -64 76 1 -64 77 1 -64 78 1 -64 79 1 -64 80 1 -64 81 1 -64 82 1 -64 83 1 -64 84 1 -64 85 1 -64 86 1 -64 87 1 -64 88 1 -64 89 1 -64 90 1 -64 91 1 -64 92 1 -64 93 1 -64 94 1 -64 95 1 -64 96 1 -64 97 1 -64 98 1 -64 99 1 -64 100 1 -65 0 1 -65 1 1 -65 2 1 -65 3 1 -65 4 1 -65 5 1 -65 6 1 -65 7 1 -65 8 1 -65 9 1 -65 10 1 -65 11 1 -65 12 1 -65 13 1 -65 14 1 -65 15 1 -65 16 1 -65 17 1 -65 18 1 -65 19 1 -65 20 1 -65 21 1 -65 22 1 -65 23 1 -65 24 1 -65 25 1 -65 26 1 -65 27 1 -65 28 1 -65 29 1 -65 30 1 -65 31 1 -65 32 1 -65 33 1 -65 34 1 -65 35 1 -65 36 1 -65 37 1 -65 38 1 -65 39 1 -65 40 1 -65 41 1 -65 42 1 -65 43 1 -65 44 1 -65 45 1 -65 46 1 -65 47 1 -65 48 1 -65 49 1 -65 50 1 -65 51 1 -65 52 1 -65 53 1 -65 54 1 -65 55 1 -65 56 1 -65 57 1 -65 58 1 -65 59 1 -65 60 1 -65 61 1 -65 62 1 -65 63 1 -65 64 1 -65 65 1 -65 66 1 -65 67 1 -65 68 1 -65 69 1 -65 70 1 -65 71 1 -65 72 1 -65 73 1 -65 74 1 -65 75 1 -65 76 1 -65 77 1 -65 78 1 -65 79 1 -65 80 1 -65 81 1 -65 82 1 -65 83 1 -65 84 1 -65 85 1 -65 86 1 -65 87 1 -65 88 1 -65 89 1 -65 90 1 -65 91 1 -65 92 1 -65 93 1 -65 94 1 -65 95 1 -65 96 1 -65 97 1 -65 98 1 -65 99 1 -65 100 1 -66 0 1 -66 1 1 -66 2 1 -66 3 1 -66 4 1 -66 5 1 -66 6 1 -66 7 1 -66 8 1 -66 9 1 -66 10 1 -66 11 1 -66 12 1 -66 13 1 -66 14 1 -66 15 1 -66 16 1 -66 17 1 -66 18 1 -66 19 1 -66 20 1 -66 21 1 -66 22 1 -66 23 1 -66 24 1 -66 25 1 -66 26 1 -66 27 1 -66 28 1 -66 29 1 -66 30 1 -66 31 1 -66 32 1 -66 33 1 -66 34 1 -66 35 1 -66 36 1 -66 37 1 -66 38 1 -66 39 1 -66 40 1 -66 41 1 -66 42 1 -66 43 1 -66 44 1 -66 45 1 -66 46 1 -66 47 1 -66 48 1 -66 49 1 -66 50 1 -66 51 1 -66 52 1 -66 53 1 -66 54 1 -66 55 1 -66 56 1 -66 57 1 -66 58 1 -66 59 1 -66 60 1 -66 61 1 -66 62 1 -66 63 1 -66 64 1 -66 65 1 -66 66 1 -66 67 1 -66 68 1 -66 69 1 -66 70 1 -66 71 1 -66 72 1 -66 73 1 -66 74 1 -66 75 1 -66 76 1 -66 77 1 -66 78 1 -66 79 1 -66 80 1 -66 81 1 -66 82 1 -66 83 1 -66 84 1 -66 85 1 -66 86 1 -66 87 1 -66 88 1 -66 89 1 -66 90 1 -66 91 1 -66 92 1 -66 93 1 -66 94 1 -66 95 1 -66 96 1 -66 97 1 -66 98 1 -66 99 1 -66 100 1 -67 0 1 -67 1 1 -67 2 1 -67 3 1 -67 4 1 -67 5 1 -67 6 1 -67 7 1 -67 8 1 -67 9 1 -67 10 1 -67 11 1 -67 12 1 -67 13 1 -67 14 1 -67 15 1 -67 16 1 -67 17 1 -67 18 1 -67 19 1 -67 20 1 -67 21 1 -67 22 1 -67 23 1 -67 24 1 -67 25 1 -67 26 1 -67 27 1 -67 28 1 -67 29 1 -67 30 1 -67 31 1 -67 32 1 -67 33 1 -67 34 1 -67 35 1 -67 36 1 -67 37 1 -67 38 1 -67 39 1 -67 40 1 -67 41 1 -67 42 1 -67 43 1 -67 44 1 -67 45 1 -67 46 1 -67 47 1 -67 48 1 -67 49 1 -67 50 1 -67 51 1 -67 52 1 -67 53 1 -67 54 1 -67 55 1 -67 56 1 -67 57 1 -67 58 1 -67 59 1 -67 60 1 -67 61 1 -67 62 1 -67 63 1 -67 64 1 -67 65 1 -67 66 1 -67 67 1 -67 68 1 -67 69 1 -67 70 1 -67 71 1 -67 72 1 -67 73 1 -67 74 1 -67 75 1 -67 76 1 -67 77 1 -67 78 1 -67 79 1 -67 80 1 -67 81 1 -67 82 1 -67 83 1 -67 84 1 -67 85 1 -67 86 1 -67 87 1 -67 88 1 -67 89 1 -67 90 1 -67 91 1 -67 92 1 -67 93 1 -67 94 1 -67 95 1 -67 96 1 -67 97 1 -67 98 1 -67 99 1 -67 100 1 -68 0 1 -68 1 1 -68 2 1 -68 3 1 -68 4 1 -68 5 1 -68 6 1 -68 7 1 -68 8 1 -68 9 1 -68 10 1 -68 11 1 -68 12 1 -68 13 1 -68 14 1 -68 15 1 -68 16 1 -68 17 1 -68 18 1 -68 19 1 -68 20 1 -68 21 1 -68 22 1 -68 23 1 -68 24 1 -68 25 1 -68 26 1 -68 27 1 -68 28 1 -68 29 1 -68 30 1 -68 31 1 -68 32 1 -68 33 1 -68 34 1 -68 35 1 -68 36 1 -68 37 1 -68 38 1 -68 39 1 -68 40 1 -68 41 1 -68 42 1 -68 43 1 -68 44 1 -68 45 1 -68 46 1 -68 47 1 -68 48 1 -68 49 1 -68 50 1 -68 51 1 -68 52 1 -68 53 1 -68 54 1 -68 55 1 -68 56 1 -68 57 1 -68 58 1 -68 59 1 -68 60 1 -68 61 1 -68 62 1 -68 63 1 -68 64 1 -68 65 1 -68 66 1 -68 67 1 -68 68 1 -68 69 1 -68 70 1 -68 71 1 -68 72 1 -68 73 1 -68 74 1 -68 75 1 -68 76 1 -68 77 1 -68 78 1 -68 79 1 -68 80 1 -68 81 1 -68 82 1 -68 83 1 -68 84 1 -68 85 1 -68 86 1 -68 87 1 -68 88 1 -68 89 1 -68 90 1 -68 91 1 -68 92 1 -68 93 1 -68 94 1 -68 95 1 -68 96 1 -68 97 1 -68 98 1 -68 99 1 -68 100 1 -69 0 1 -69 1 1 -69 2 1 -69 3 1 -69 4 1 -69 5 1 -69 6 1 -69 7 1 -69 8 1 -69 9 1 -69 10 1 -69 11 1 -69 12 1 -69 13 1 -69 14 1 -69 15 1 -69 16 1 -69 17 1 -69 18 1 -69 19 1 -69 20 1 -69 21 1 -69 22 1 -69 23 1 -69 24 1 -69 25 1 -69 26 1 -69 27 1 -69 28 1 -69 29 1 -69 30 1 -69 31 1 -69 32 1 -69 33 1 -69 34 1 -69 35 1 -69 36 1 -69 37 1 -69 38 1 -69 39 1 -69 40 1 -69 41 1 -69 42 1 -69 43 1 -69 44 1 -69 45 1 -69 46 1 -69 47 1 -69 48 1 -69 49 1 -69 50 1 -69 51 1 -69 52 1 -69 53 1 -69 54 1 -69 55 1 -69 56 1 -69 57 1 -69 58 1 -69 59 1 -69 60 1 -69 61 1 -69 62 1 -69 63 1 -69 64 1 -69 65 1 -69 66 1 -69 67 1 -69 68 1 -69 69 1 -69 70 1 -69 71 1 -69 72 1 -69 73 1 -69 74 1 -69 75 1 -69 76 1 -69 77 1 -69 78 1 -69 79 1 -69 80 1 -69 81 1 -69 82 1 -69 83 1 -69 84 1 -69 85 1 -69 86 1 -69 87 1 -69 88 1 -69 89 1 -69 90 1 -69 91 1 -69 92 1 -69 93 1 -69 94 1 -69 95 1 -69 96 1 -69 97 1 -69 98 1 -69 99 1 -69 100 1 -70 0 1 -70 1 1 -70 2 1 -70 3 1 -70 4 1 -70 5 1 -70 6 1 -70 7 1 -70 8 1 -70 9 1 -70 10 1 -70 11 1 -70 12 1 -70 13 1 -70 14 1 -70 15 1 -70 16 1 -70 17 1 -70 18 1 -70 19 1 -70 20 1 -70 21 1 -70 22 1 -70 23 1 -70 24 1 -70 25 1 -70 26 1 -70 27 1 -70 28 1 -70 29 1 -70 30 1 -70 31 1 -70 32 1 -70 33 1 -70 34 1 -70 35 1 -70 36 1 -70 37 1 -70 38 1 -70 39 1 -70 40 1 -70 41 1 -70 42 1 -70 43 1 -70 44 1 -70 45 1 -70 46 1 -70 47 1 -70 48 1 -70 49 1 -70 50 1 -70 51 1 -70 52 1 -70 53 1 -70 54 1 -70 55 1 -70 56 1 -70 57 1 -70 58 1 -70 59 1 -70 60 1 -70 61 1 -70 62 1 -70 63 1 -70 64 1 -70 65 1 -70 66 1 -70 67 1 -70 68 1 -70 69 1 -70 70 1 -70 71 1 -70 72 1 -70 73 1 -70 74 1 -70 75 1 -70 76 1 -70 77 1 -70 78 1 -70 79 1 -70 80 1 -70 81 1 -70 82 1 -70 83 1 -70 84 1 -70 85 1 -70 86 1 -70 87 1 -70 88 1 -70 89 1 -70 90 1 -70 91 1 -70 92 1 -70 93 1 -70 94 1 -70 95 1 -70 96 1 -70 97 1 -70 98 1 -70 99 1 -70 100 1 -71 0 1 -71 1 1 -71 2 1 -71 3 1 -71 4 1 -71 5 1 -71 6 1 -71 7 1 -71 8 1 -71 9 1 -71 10 1 -71 11 1 -71 12 1 -71 13 1 -71 14 1 -71 15 1 -71 16 1 -71 17 1 -71 18 1 -71 19 1 -71 20 1 -71 21 1 -71 22 1 -71 23 1 -71 24 1 -71 25 1 -71 26 1 -71 27 1 -71 28 1 -71 29 1 -71 30 1 -71 31 1 -71 32 1 -71 33 1 -71 34 1 -71 35 1 -71 36 1 -71 37 1 -71 38 1 -71 39 1 -71 40 1 -71 41 1 -71 42 1 -71 43 1 -71 44 1 -71 45 1 -71 46 1 -71 47 1 -71 48 1 -71 49 1 -71 50 1 -71 51 1 -71 52 1 -71 53 1 -71 54 1 -71 55 1 -71 56 1 -71 57 1 -71 58 1 -71 59 1 -71 60 1 -71 61 1 -71 62 1 -71 63 1 -71 64 1 -71 65 1 -71 66 1 -71 67 1 -71 68 1 -71 69 1 -71 70 1 -71 71 1 -71 72 1 -71 73 1 -71 74 1 -71 75 1 -71 76 1 -71 77 1 -71 78 1 -71 79 1 -71 80 1 -71 81 1 -71 82 1 -71 83 1 -71 84 1 -71 85 1 -71 86 1 -71 87 1 -71 88 1 -71 89 1 -71 90 1 -71 91 1 -71 92 1 -71 93 1 -71 94 1 -71 95 1 -71 96 1 -71 97 1 -71 98 1 -71 99 1 -71 100 1 -72 0 1 -72 1 1 -72 2 1 -72 3 1 -72 4 1 -72 5 1 -72 6 1 -72 7 1 -72 8 1 -72 9 1 -72 10 1 -72 11 1 -72 12 1 -72 13 1 -72 14 1 -72 15 1 -72 16 1 -72 17 1 -72 18 1 -72 19 1 -72 20 1 -72 21 1 -72 22 1 -72 23 1 -72 24 1 -72 25 1 -72 26 1 -72 27 1 -72 28 1 -72 29 1 -72 30 1 -72 31 1 -72 32 1 -72 33 1 -72 34 1 -72 35 1 -72 36 1 -72 37 1 -72 38 1 -72 39 1 -72 40 1 -72 41 1 -72 42 1 -72 43 1 -72 44 1 -72 45 1 -72 46 1 -72 47 1 -72 48 1 -72 49 1 -72 50 1 -72 51 1 -72 52 1 -72 53 1 -72 54 1 -72 55 1 -72 56 1 -72 57 1 -72 58 1 -72 59 1 -72 60 1 -72 61 1 -72 62 1 -72 63 1 -72 64 1 -72 65 1 -72 66 1 -72 67 1 -72 68 1 -72 69 1 -72 70 1 -72 71 1 -72 72 1 -72 73 1 -72 74 1 -72 75 1 -72 76 1 -72 77 1 -72 78 1 -72 79 1 -72 80 1 -72 81 1 -72 82 1 -72 83 1 -72 84 1 -72 85 1 -72 86 1 -72 87 1 -72 88 1 -72 89 1 -72 90 1 -72 91 1 -72 92 1 -72 93 1 -72 94 1 -72 95 1 -72 96 1 -72 97 1 -72 98 1 -72 99 1 -72 100 1 -73 0 1 -73 1 1 -73 2 1 -73 3 1 -73 4 1 -73 5 1 -73 6 1 -73 7 1 -73 8 1 -73 9 1 -73 10 1 -73 11 1 -73 12 1 -73 13 1 -73 14 1 -73 15 1 -73 16 1 -73 17 1 -73 18 1 -73 19 1 -73 20 1 -73 21 1 -73 22 1 -73 23 1 -73 24 1 -73 25 1 -73 26 1 -73 27 1 -73 28 1 -73 29 1 -73 30 1 -73 31 1 -73 32 1 -73 33 1 -73 34 1 -73 35 1 -73 36 1 -73 37 1 -73 38 1 -73 39 1 -73 40 1 -73 41 1 -73 42 1 -73 43 1 -73 44 1 -73 45 1 -73 46 1 -73 47 1 -73 48 1 -73 49 1 -73 50 1 -73 51 1 -73 52 1 -73 53 1 -73 54 1 -73 55 1 -73 56 1 -73 57 1 -73 58 1 -73 59 1 -73 60 1 -73 61 1 -73 62 1 -73 63 1 -73 64 1 -73 65 1 -73 66 1 -73 67 1 -73 68 1 -73 69 1 -73 70 1 -73 71 1 -73 72 1 -73 73 1 -73 74 1 -73 75 1 -73 76 1 -73 77 1 -73 78 1 -73 79 1 -73 80 1 -73 81 1 -73 82 1 -73 83 1 -73 84 1 -73 85 1 -73 86 1 -73 87 1 -73 88 1 -73 89 1 -73 90 1 -73 91 1 -73 92 1 -73 93 1 -73 94 1 -73 95 1 -73 96 1 -73 97 1 -73 98 1 -73 99 1 -73 100 1 -74 0 1 -74 1 1 -74 2 1 -74 3 1 -74 4 1 -74 5 1 -74 6 1 -74 7 1 -74 8 1 -74 9 1 -74 10 1 -74 11 1 -74 12 1 -74 13 1 -74 14 1 -74 15 1 -74 16 1 -74 17 1 -74 18 1 -74 19 1 -74 20 1 -74 21 1 -74 22 1 -74 23 1 -74 24 1 -74 25 1 -74 26 1 -74 27 1 -74 28 1 -74 29 1 -74 30 1 -74 31 1 -74 32 1 -74 33 1 -74 34 1 -74 35 1 -74 36 1 -74 37 1 -74 38 1 -74 39 1 -74 40 1 -74 41 1 -74 42 1 -74 43 1 -74 44 1 -74 45 1 -74 46 1 -74 47 1 -74 48 1 -74 49 1 -74 50 1 -74 51 1 -74 52 1 -74 53 1 -74 54 1 -74 55 1 -74 56 1 -74 57 1 -74 58 1 -74 59 1 -74 60 1 -74 61 1 -74 62 1 -74 63 1 -74 64 1 -74 65 1 -74 66 1 -74 67 1 -74 68 1 -74 69 1 -74 70 1 -74 71 1 -74 72 1 -74 73 1 -74 74 1 -74 75 1 -74 76 1 -74 77 1 -74 78 1 -74 79 1 -74 80 1 -74 81 1 -74 82 1 -74 83 1 -74 84 1 -74 85 1 -74 86 1 -74 87 1 -74 88 1 -74 89 1 -74 90 1 -74 91 1 -74 92 1 -74 93 1 -74 94 1 -74 95 1 -74 96 1 -74 97 1 -74 98 1 -74 99 1 -74 100 1 -75 0 1 -75 1 1 -75 2 1 -75 3 1 -75 4 1 -75 5 1 -75 6 1 -75 7 1 -75 8 1 -75 9 1 -75 10 1 -75 11 1 -75 12 1 -75 13 1 -75 14 1 -75 15 1 -75 16 1 -75 17 1 -75 18 1 -75 19 1 -75 20 1 -75 21 1 -75 22 1 -75 23 1 -75 24 1 -75 25 1 -75 26 1 -75 27 1 -75 28 1 -75 29 1 -75 30 1 -75 31 1 -75 32 1 -75 33 1 -75 34 1 -75 35 1 -75 36 1 -75 37 1 -75 38 1 -75 39 1 -75 40 1 -75 41 1 -75 42 1 -75 43 1 -75 44 1 -75 45 1 -75 46 1 -75 47 1 -75 48 1 -75 49 1 -75 50 1 -75 51 1 -75 52 1 -75 53 1 -75 54 1 -75 55 1 -75 56 1 -75 57 1 -75 58 1 -75 59 1 -75 60 1 -75 61 1 -75 62 1 -75 63 1 -75 64 1 -75 65 1 -75 66 1 -75 67 1 -75 68 1 -75 69 1 -75 70 1 -75 71 1 -75 72 1 -75 73 1 -75 74 1 -75 75 1 -75 76 1 -75 77 1 -75 78 1 -75 79 1 -75 80 1 -75 81 1 -75 82 1 -75 83 1 -75 84 1 -75 85 1 -75 86 1 -75 87 1 -75 88 1 -75 89 1 -75 90 1 -75 91 1 -75 92 1 -75 93 1 -75 94 1 -75 95 1 -75 96 1 -75 97 1 -75 98 1 -75 99 1 -75 100 1 -76 0 1 -76 1 1 -76 2 1 -76 3 1 -76 4 1 -76 5 1 -76 6 1 -76 7 1 -76 8 1 -76 9 1 -76 10 1 -76 11 1 -76 12 1 -76 13 1 -76 14 1 -76 15 1 -76 16 1 -76 17 1 -76 18 1 -76 19 1 -76 20 1 -76 21 1 -76 22 1 -76 23 1 -76 24 1 -76 25 1 -76 26 1 -76 27 1 -76 28 1 -76 29 1 -76 30 1 -76 31 1 -76 32 1 -76 33 1 -76 34 1 -76 35 1 -76 36 1 -76 37 1 -76 38 1 -76 39 1 -76 40 1 -76 41 1 -76 42 1 -76 43 1 -76 44 1 -76 45 1 -76 46 1 -76 47 1 -76 48 1 -76 49 1 -76 50 1 -76 51 1 -76 52 1 -76 53 1 -76 54 1 -76 55 1 -76 56 1 -76 57 1 -76 58 1 -76 59 1 -76 60 1 -76 61 1 -76 62 1 -76 63 1 -76 64 1 -76 65 1 -76 66 1 -76 67 1 -76 68 1 -76 69 1 -76 70 1 -76 71 1 -76 72 1 -76 73 1 -76 74 1 -76 75 1 -76 76 1 -76 77 1 -76 78 1 -76 79 1 -76 80 1 -76 81 1 -76 82 1 -76 83 1 -76 84 1 -76 85 1 -76 86 1 -76 87 1 -76 88 1 -76 89 1 -76 90 1 -76 91 1 -76 92 1 -76 93 1 -76 94 1 -76 95 1 -76 96 1 -76 97 1 -76 98 1 -76 99 1 -76 100 1 -77 0 1 -77 1 1 -77 2 1 -77 3 1 -77 4 1 -77 5 1 -77 6 1 -77 7 1 -77 8 1 -77 9 1 -77 10 1 -77 11 1 -77 12 1 -77 13 1 -77 14 1 -77 15 1 -77 16 1 -77 17 1 -77 18 1 -77 19 1 -77 20 1 -77 21 1 -77 22 1 -77 23 1 -77 24 1 -77 25 1 -77 26 1 -77 27 1 -77 28 1 -77 29 1 -77 30 1 -77 31 1 -77 32 1 -77 33 1 -77 34 1 -77 35 1 -77 36 1 -77 37 1 -77 38 1 -77 39 1 -77 40 1 -77 41 1 -77 42 1 -77 43 1 -77 44 1 -77 45 1 -77 46 1 -77 47 1 -77 48 1 -77 49 1 -77 50 1 -77 51 1 -77 52 1 -77 53 1 -77 54 1 -77 55 1 -77 56 1 -77 57 1 -77 58 1 -77 59 1 -77 60 1 -77 61 1 -77 62 1 -77 63 1 -77 64 1 -77 65 1 -77 66 1 -77 67 1 -77 68 1 -77 69 1 -77 70 1 -77 71 1 -77 72 1 -77 73 1 -77 74 1 -77 75 1 -77 76 1 -77 77 1 -77 78 1 -77 79 1 -77 80 1 -77 81 1 -77 82 1 -77 83 1 -77 84 1 -77 85 1 -77 86 1 -77 87 1 -77 88 1 -77 89 1 -77 90 1 -77 91 1 -77 92 1 -77 93 1 -77 94 1 -77 95 1 -77 96 1 -77 97 1 -77 98 1 -77 99 1 -77 100 1 -78 0 1 -78 1 1 -78 2 1 -78 3 1 -78 4 1 -78 5 1 -78 6 1 -78 7 1 -78 8 1 -78 9 1 -78 10 1 -78 11 1 -78 12 1 -78 13 1 -78 14 1 -78 15 1 -78 16 1 -78 17 1 -78 18 1 -78 19 1 -78 20 1 -78 21 1 -78 22 1 -78 23 1 -78 24 1 -78 25 1 -78 26 1 -78 27 1 -78 28 1 -78 29 1 -78 30 1 -78 31 1 -78 32 1 -78 33 1 -78 34 1 -78 35 1 -78 36 1 -78 37 1 -78 38 1 -78 39 1 -78 40 1 -78 41 1 -78 42 1 -78 43 1 -78 44 1 -78 45 1 -78 46 1 -78 47 1 -78 48 1 -78 49 1 -78 50 1 -78 51 1 -78 52 1 -78 53 1 -78 54 1 -78 55 1 -78 56 1 -78 57 1 -78 58 1 -78 59 1 -78 60 1 -78 61 1 -78 62 1 -78 63 1 -78 64 1 -78 65 1 -78 66 1 -78 67 1 -78 68 1 -78 69 1 -78 70 1 -78 71 1 -78 72 1 -78 73 1 -78 74 1 -78 75 1 -78 76 1 -78 77 1 -78 78 1 -78 79 1 -78 80 1 -78 81 1 -78 82 1 -78 83 1 -78 84 1 -78 85 1 -78 86 1 -78 87 1 -78 88 1 -78 89 1 -78 90 1 -78 91 1 -78 92 1 -78 93 1 -78 94 1 -78 95 1 -78 96 1 -78 97 1 -78 98 1 -78 99 1 -78 100 1 -79 0 1 -79 1 1 -79 2 1 -79 3 1 -79 4 1 -79 5 1 -79 6 1 -79 7 1 -79 8 1 -79 9 1 -79 10 1 -79 11 1 -79 12 1 -79 13 1 -79 14 1 -79 15 1 -79 16 1 -79 17 1 -79 18 1 -79 19 1 -79 20 1 -79 21 1 -79 22 1 -79 23 1 -79 24 1 -79 25 1 -79 26 1 -79 27 1 -79 28 1 -79 29 1 -79 30 1 -79 31 1 -79 32 1 -79 33 1 -79 34 1 -79 35 1 -79 36 1 -79 37 1 -79 38 1 -79 39 1 -79 40 1 -79 41 1 -79 42 1 -79 43 1 -79 44 1 -79 45 1 -79 46 1 -79 47 1 -79 48 1 -79 49 1 -79 50 1 -79 51 1 -79 52 1 -79 53 1 -79 54 1 -79 55 1 -79 56 1 -79 57 1 -79 58 1 -79 59 1 -79 60 1 -79 61 1 -79 62 1 -79 63 1 -79 64 1 -79 65 1 -79 66 1 -79 67 1 -79 68 1 -79 69 1 -79 70 1 -79 71 1 -79 72 1 -79 73 1 -79 74 1 -79 75 1 -79 76 1 -79 77 1 -79 78 1 -79 79 1 -79 80 1 -79 81 1 -79 82 1 -79 83 1 -79 84 1 -79 85 1 -79 86 1 -79 87 1 -79 88 1 -79 89 1 -79 90 1 -79 91 1 -79 92 1 -79 93 1 -79 94 1 -79 95 1 -79 96 1 -79 97 1 -79 98 1 -79 99 1 -79 100 1 -80 0 1 -80 1 1 -80 2 1 -80 3 1 -80 4 1 -80 5 1 -80 6 1 -80 7 1 -80 8 1 -80 9 1 -80 10 1 -80 11 1 -80 12 1 -80 13 1 -80 14 1 -80 15 1 -80 16 1 -80 17 1 -80 18 1 -80 19 1 -80 20 1 -80 21 1 -80 22 1 -80 23 1 -80 24 1 -80 25 1 -80 26 1 -80 27 1 -80 28 1 -80 29 1 -80 30 1 -80 31 1 -80 32 1 -80 33 1 -80 34 1 -80 35 1 -80 36 1 -80 37 1 -80 38 1 -80 39 1 -80 40 1 -80 41 1 -80 42 1 -80 43 1 -80 44 1 -80 45 1 -80 46 1 -80 47 1 -80 48 1 -80 49 1 -80 50 1 -80 51 1 -80 52 1 -80 53 1 -80 54 1 -80 55 1 -80 56 1 -80 57 1 -80 58 1 -80 59 1 -80 60 1 -80 61 1 -80 62 1 -80 63 1 -80 64 1 -80 65 1 -80 66 1 -80 67 1 -80 68 1 -80 69 1 -80 70 1 -80 71 1 -80 72 1 -80 73 1 -80 74 1 -80 75 1 -80 76 1 -80 77 1 -80 78 1 -80 79 1 -80 80 1 -80 81 1 -80 82 1 -80 83 1 -80 84 1 -80 85 1 -80 86 1 -80 87 1 -80 88 1 -80 89 1 -80 90 1 -80 91 1 -80 92 1 -80 93 1 -80 94 1 -80 95 1 -80 96 1 -80 97 1 -80 98 1 -80 99 1 -80 100 1 -81 0 1 -81 1 1 -81 2 1 -81 3 1 -81 4 1 -81 5 1 -81 6 1 -81 7 1 -81 8 1 -81 9 1 -81 10 1 -81 11 1 -81 12 1 -81 13 1 -81 14 1 -81 15 1 -81 16 1 -81 17 1 -81 18 1 -81 19 1 -81 20 1 -81 21 1 -81 22 1 -81 23 1 -81 24 1 -81 25 1 -81 26 1 -81 27 1 -81 28 1 -81 29 1 -81 30 1 -81 31 1 -81 32 1 -81 33 1 -81 34 1 -81 35 1 -81 36 1 -81 37 1 -81 38 1 -81 39 1 -81 40 1 -81 41 1 -81 42 1 -81 43 1 -81 44 1 -81 45 1 -81 46 1 -81 47 1 -81 48 1 -81 49 1 -81 50 1 -81 51 1 -81 52 1 -81 53 1 -81 54 1 -81 55 1 -81 56 1 -81 57 1 -81 58 1 -81 59 1 -81 60 1 -81 61 1 -81 62 1 -81 63 1 -81 64 1 -81 65 1 -81 66 1 -81 67 1 -81 68 1 -81 69 1 -81 70 1 -81 71 1 -81 72 1 -81 73 1 -81 74 1 -81 75 1 -81 76 1 -81 77 1 -81 78 1 -81 79 1 -81 80 1 -81 81 1 -81 82 1 -81 83 1 -81 84 1 -81 85 1 -81 86 1 -81 87 1 -81 88 1 -81 89 1 -81 90 1 -81 91 1 -81 92 1 -81 93 1 -81 94 1 -81 95 1 -81 96 1 -81 97 1 -81 98 1 -81 99 1 -81 100 1 -82 0 1 -82 1 1 -82 2 1 -82 3 1 -82 4 1 -82 5 1 -82 6 1 -82 7 1 -82 8 1 -82 9 1 -82 10 1 -82 11 1 -82 12 1 -82 13 1 -82 14 1 -82 15 1 -82 16 1 -82 17 1 -82 18 1 -82 19 1 -82 20 1 -82 21 1 -82 22 1 -82 23 1 -82 24 1 -82 25 1 -82 26 1 -82 27 1 -82 28 1 -82 29 1 -82 30 1 -82 31 1 -82 32 1 -82 33 1 -82 34 1 -82 35 1 -82 36 1 -82 37 1 -82 38 1 -82 39 1 -82 40 1 -82 41 1 -82 42 1 -82 43 1 -82 44 1 -82 45 1 -82 46 1 -82 47 1 -82 48 1 -82 49 1 -82 50 1 -82 51 1 -82 52 1 -82 53 1 -82 54 1 -82 55 1 -82 56 1 -82 57 1 -82 58 1 -82 59 1 -82 60 1 -82 61 1 -82 62 1 -82 63 1 -82 64 1 -82 65 1 -82 66 1 -82 67 1 -82 68 1 -82 69 1 -82 70 1 -82 71 1 -82 72 1 -82 73 1 -82 74 1 -82 75 1 -82 76 1 -82 77 1 -82 78 1 -82 79 1 -82 80 1 -82 81 1 -82 82 1 -82 83 1 -82 84 1 -82 85 1 -82 86 1 -82 87 1 -82 88 1 -82 89 1 -82 90 1 -82 91 1 -82 92 1 -82 93 1 -82 94 1 -82 95 1 -82 96 1 -82 97 1 -82 98 1 -82 99 1 -82 100 1 -83 0 1 -83 1 1 -83 2 1 -83 3 1 -83 4 1 -83 5 1 -83 6 1 -83 7 1 -83 8 1 -83 9 1 -83 10 1 -83 11 1 -83 12 1 -83 13 1 -83 14 1 -83 15 1 -83 16 1 -83 17 1 -83 18 1 -83 19 1 -83 20 1 -83 21 1 -83 22 1 -83 23 1 -83 24 1 -83 25 1 -83 26 1 -83 27 1 -83 28 1 -83 29 1 -83 30 1 -83 31 1 -83 32 1 -83 33 1 -83 34 1 -83 35 1 -83 36 1 -83 37 1 -83 38 1 -83 39 1 -83 40 1 -83 41 1 -83 42 1 -83 43 1 -83 44 1 -83 45 1 -83 46 1 -83 47 1 -83 48 1 -83 49 1 -83 50 1 -83 51 1 -83 52 1 -83 53 1 -83 54 1 -83 55 1 -83 56 1 -83 57 1 -83 58 1 -83 59 1 -83 60 1 -83 61 1 -83 62 1 -83 63 1 -83 64 1 -83 65 1 -83 66 1 -83 67 1 -83 68 1 -83 69 1 -83 70 1 -83 71 1 -83 72 1 -83 73 1 -83 74 1 -83 75 1 -83 76 1 -83 77 1 -83 78 1 -83 79 1 -83 80 1 -83 81 1 -83 82 1 -83 83 1 -83 84 1 -83 85 1 -83 86 1 -83 87 1 -83 88 1 -83 89 1 -83 90 1 -83 91 1 -83 92 1 -83 93 1 -83 94 1 -83 95 1 -83 96 1 -83 97 1 -83 98 1 -83 99 1 -83 100 1 -84 0 1 -84 1 1 -84 2 1 -84 3 1 -84 4 1 -84 5 1 -84 6 1 -84 7 1 -84 8 1 -84 9 1 -84 10 1 -84 11 1 -84 12 1 -84 13 1 -84 14 1 -84 15 1 -84 16 1 -84 17 1 -84 18 1 -84 19 1 -84 20 1 -84 21 1 -84 22 1 -84 23 1 -84 24 1 -84 25 1 -84 26 1 -84 27 1 -84 28 1 -84 29 1 -84 30 1 -84 31 1 -84 32 1 -84 33 1 -84 34 1 -84 35 1 -84 36 1 -84 37 1 -84 38 1 -84 39 1 -84 40 1 -84 41 1 -84 42 1 -84 43 1 -84 44 1 -84 45 1 -84 46 1 -84 47 1 -84 48 1 -84 49 1 -84 50 1 -84 51 1 -84 52 1 -84 53 1 -84 54 1 -84 55 1 -84 56 1 -84 57 1 -84 58 1 -84 59 1 -84 60 1 -84 61 1 -84 62 1 -84 63 1 -84 64 1 -84 65 1 -84 66 1 -84 67 1 -84 68 1 -84 69 1 -84 70 1 -84 71 1 -84 72 1 -84 73 1 -84 74 1 -84 75 1 -84 76 1 -84 77 1 -84 78 1 -84 79 1 -84 80 1 -84 81 1 -84 82 1 -84 83 1 -84 84 1 -84 85 1 -84 86 1 -84 87 1 -84 88 1 -84 89 1 -84 90 1 -84 91 1 -84 92 1 -84 93 1 -84 94 1 -84 95 1 -84 96 1 -84 97 1 -84 98 1 -84 99 1 -84 100 1 -85 0 1 -85 1 1 -85 2 1 -85 3 1 -85 4 1 -85 5 1 -85 6 1 -85 7 1 -85 8 1 -85 9 1 -85 10 1 -85 11 1 -85 12 1 -85 13 1 -85 14 1 -85 15 1 -85 16 1 -85 17 1 -85 18 1 -85 19 1 -85 20 1 -85 21 1 -85 22 1 -85 23 1 -85 24 1 -85 25 1 -85 26 1 -85 27 1 -85 28 1 -85 29 1 -85 30 1 -85 31 1 -85 32 1 -85 33 1 -85 34 1 -85 35 1 -85 36 1 -85 37 1 -85 38 1 -85 39 1 -85 40 1 -85 41 1 -85 42 1 -85 43 1 -85 44 1 -85 45 1 -85 46 1 -85 47 1 -85 48 1 -85 49 1 -85 50 1 -85 51 1 -85 52 1 -85 53 1 -85 54 1 -85 55 1 -85 56 1 -85 57 1 -85 58 1 -85 59 1 -85 60 1 -85 61 1 -85 62 1 -85 63 1 -85 64 1 -85 65 1 -85 66 1 -85 67 1 -85 68 1 -85 69 1 -85 70 1 -85 71 1 -85 72 1 -85 73 1 -85 74 1 -85 75 1 -85 76 1 -85 77 1 -85 78 1 -85 79 1 -85 80 1 -85 81 1 -85 82 1 -85 83 1 -85 84 1 -85 85 1 -85 86 1 -85 87 1 -85 88 1 -85 89 1 -85 90 1 -85 91 1 -85 92 1 -85 93 1 -85 94 1 -85 95 1 -85 96 1 -85 97 1 -85 98 1 -85 99 1 -85 100 1 -86 0 1 -86 1 1 -86 2 1 -86 3 1 -86 4 1 -86 5 1 -86 6 1 -86 7 1 -86 8 1 -86 9 1 -86 10 1 -86 11 1 -86 12 1 -86 13 1 -86 14 1 -86 15 1 -86 16 1 -86 17 1 -86 18 1 -86 19 1 -86 20 1 -86 21 1 -86 22 1 -86 23 1 -86 24 1 -86 25 1 -86 26 1 -86 27 1 -86 28 1 -86 29 1 -86 30 1 -86 31 1 -86 32 1 -86 33 1 -86 34 1 -86 35 1 -86 36 1 -86 37 1 -86 38 1 -86 39 1 -86 40 1 -86 41 1 -86 42 1 -86 43 1 -86 44 1 -86 45 1 -86 46 1 -86 47 1 -86 48 1 -86 49 1 -86 50 1 -86 51 1 -86 52 1 -86 53 1 -86 54 1 -86 55 1 -86 56 1 -86 57 1 -86 58 1 -86 59 1 -86 60 1 -86 61 1 -86 62 1 -86 63 1 -86 64 1 -86 65 1 -86 66 1 -86 67 1 -86 68 1 -86 69 1 -86 70 1 -86 71 1 -86 72 1 -86 73 1 -86 74 1 -86 75 1 -86 76 1 -86 77 1 -86 78 1 -86 79 1 -86 80 1 -86 81 1 -86 82 1 -86 83 1 -86 84 1 -86 85 1 -86 86 1 -86 87 1 -86 88 1 -86 89 1 -86 90 1 -86 91 1 -86 92 1 -86 93 1 -86 94 1 -86 95 1 -86 96 1 -86 97 1 -86 98 1 -86 99 1 -86 100 1 -87 0 1 -87 1 1 -87 2 1 -87 3 1 -87 4 1 -87 5 1 -87 6 1 -87 7 1 -87 8 1 -87 9 1 -87 10 1 -87 11 1 -87 12 1 -87 13 1 -87 14 1 -87 15 1 -87 16 1 -87 17 1 -87 18 1 -87 19 1 -87 20 1 -87 21 1 -87 22 1 -87 23 1 -87 24 1 -87 25 1 -87 26 1 -87 27 1 -87 28 1 -87 29 1 -87 30 1 -87 31 1 -87 32 1 -87 33 1 -87 34 1 -87 35 1 -87 36 1 -87 37 1 -87 38 1 -87 39 1 -87 40 1 -87 41 1 -87 42 1 -87 43 1 -87 44 1 -87 45 1 -87 46 1 -87 47 1 -87 48 1 -87 49 1 -87 50 1 -87 51 1 -87 52 1 -87 53 1 -87 54 1 -87 55 1 -87 56 1 -87 57 1 -87 58 1 -87 59 1 -87 60 1 -87 61 1 -87 62 1 -87 63 1 -87 64 1 -87 65 1 -87 66 1 -87 67 1 -87 68 1 -87 69 1 -87 70 1 -87 71 1 -87 72 1 -87 73 1 -87 74 1 -87 75 1 -87 76 1 -87 77 1 -87 78 1 -87 79 1 -87 80 1 -87 81 1 -87 82 1 -87 83 1 -87 84 1 -87 85 1 -87 86 1 -87 87 1 -87 88 1 -87 89 1 -87 90 1 -87 91 1 -87 92 1 -87 93 1 -87 94 1 -87 95 1 -87 96 1 -87 97 1 -87 98 1 -87 99 1 -87 100 1 -88 0 1 -88 1 1 -88 2 1 -88 3 1 -88 4 1 -88 5 1 -88 6 1 -88 7 1 -88 8 1 -88 9 1 -88 10 1 -88 11 1 -88 12 1 -88 13 1 -88 14 1 -88 15 1 -88 16 1 -88 17 1 -88 18 1 -88 19 1 -88 20 1 -88 21 1 -88 22 1 -88 23 1 -88 24 1 -88 25 1 -88 26 1 -88 27 1 -88 28 1 -88 29 1 -88 30 1 -88 31 1 -88 32 1 -88 33 1 -88 34 1 -88 35 1 -88 36 1 -88 37 1 -88 38 1 -88 39 1 -88 40 1 -88 41 1 -88 42 1 -88 43 1 -88 44 1 -88 45 1 -88 46 1 -88 47 1 -88 48 1 -88 49 1 -88 50 1 -88 51 1 -88 52 1 -88 53 1 -88 54 1 -88 55 1 -88 56 1 -88 57 1 -88 58 1 -88 59 1 -88 60 1 -88 61 1 -88 62 1 -88 63 1 -88 64 1 -88 65 1 -88 66 1 -88 67 1 -88 68 1 -88 69 1 -88 70 1 -88 71 1 -88 72 1 -88 73 1 -88 74 1 -88 75 1 -88 76 1 -88 77 1 -88 78 1 -88 79 1 -88 80 1 -88 81 1 -88 82 1 -88 83 1 -88 84 1 -88 85 1 -88 86 1 -88 87 1 -88 88 1 -88 89 1 -88 90 1 -88 91 1 -88 92 1 -88 93 1 -88 94 1 -88 95 1 -88 96 1 -88 97 1 -88 98 1 -88 99 1 -88 100 1 -89 0 1 -89 1 1 -89 2 1 -89 3 1 -89 4 1 -89 5 1 -89 6 1 -89 7 1 -89 8 1 -89 9 1 -89 10 1 -89 11 1 -89 12 1 -89 13 1 -89 14 1 -89 15 1 -89 16 1 -89 17 1 -89 18 1 -89 19 1 -89 20 1 -89 21 1 -89 22 1 -89 23 1 -89 24 1 -89 25 1 -89 26 1 -89 27 1 -89 28 1 -89 29 1 -89 30 1 -89 31 1 -89 32 1 -89 33 1 -89 34 1 -89 35 1 -89 36 1 -89 37 1 -89 38 1 -89 39 1 -89 40 1 -89 41 1 -89 42 1 -89 43 1 -89 44 1 -89 45 1 -89 46 1 -89 47 1 -89 48 1 -89 49 1 -89 50 1 -89 51 1 -89 52 1 -89 53 1 -89 54 1 -89 55 1 -89 56 1 -89 57 1 -89 58 1 -89 59 1 -89 60 1 -89 61 1 -89 62 1 -89 63 1 -89 64 1 -89 65 1 -89 66 1 -89 67 1 -89 68 1 -89 69 1 -89 70 1 -89 71 1 -89 72 1 -89 73 1 -89 74 1 -89 75 1 -89 76 1 -89 77 1 -89 78 1 -89 79 1 -89 80 1 -89 81 1 -89 82 1 -89 83 1 -89 84 1 -89 85 1 -89 86 1 -89 87 1 -89 88 1 -89 89 1 -89 90 1 -89 91 1 -89 92 1 -89 93 1 -89 94 1 -89 95 1 -89 96 1 -89 97 1 -89 98 1 -89 99 1 -89 100 1 -90 0 1 -90 1 1 -90 2 1 -90 3 1 -90 4 1 -90 5 1 -90 6 1 -90 7 1 -90 8 1 -90 9 1 -90 10 1 -90 11 1 -90 12 1 -90 13 1 -90 14 1 -90 15 1 -90 16 1 -90 17 1 -90 18 1 -90 19 1 -90 20 1 -90 21 1 -90 22 1 -90 23 1 -90 24 1 -90 25 1 -90 26 1 -90 27 1 -90 28 1 -90 29 1 -90 30 1 -90 31 1 -90 32 1 -90 33 1 -90 34 1 -90 35 1 -90 36 1 -90 37 1 -90 38 1 -90 39 1 -90 40 1 -90 41 1 -90 42 1 -90 43 1 -90 44 1 -90 45 1 -90 46 1 -90 47 1 -90 48 1 -90 49 1 -90 50 1 -90 51 1 -90 52 1 -90 53 1 -90 54 1 -90 55 1 -90 56 1 -90 57 1 -90 58 1 -90 59 1 -90 60 1 -90 61 1 -90 62 1 -90 63 1 -90 64 1 -90 65 1 -90 66 1 -90 67 1 -90 68 1 -90 69 1 -90 70 1 -90 71 1 -90 72 1 -90 73 1 -90 74 1 -90 75 1 -90 76 1 -90 77 1 -90 78 1 -90 79 1 -90 80 1 -90 81 1 -90 82 1 -90 83 1 -90 84 1 -90 85 1 -90 86 1 -90 87 1 -90 88 1 -90 89 1 -90 90 1 -90 91 1 -90 92 1 -90 93 1 -90 94 1 -90 95 1 -90 96 1 -90 97 1 -90 98 1 -90 99 1 -90 100 1 -91 0 1 -91 1 1 -91 2 1 -91 3 1 -91 4 1 -91 5 1 -91 6 1 -91 7 1 -91 8 1 -91 9 1 -91 10 1 -91 11 1 -91 12 1 -91 13 1 -91 14 1 -91 15 1 -91 16 1 -91 17 1 -91 18 1 -91 19 1 -91 20 1 -91 21 1 -91 22 1 -91 23 1 -91 24 1 -91 25 1 -91 26 1 -91 27 1 -91 28 1 -91 29 1 -91 30 1 -91 31 1 -91 32 1 -91 33 1 -91 34 1 -91 35 1 -91 36 1 -91 37 1 -91 38 1 -91 39 1 -91 40 1 -91 41 1 -91 42 1 -91 43 1 -91 44 1 -91 45 1 -91 46 1 -91 47 1 -91 48 1 -91 49 1 -91 50 1 -91 51 1 -91 52 1 -91 53 1 -91 54 1 -91 55 1 -91 56 1 -91 57 1 -91 58 1 -91 59 1 -91 60 1 -91 61 1 -91 62 1 -91 63 1 -91 64 1 -91 65 1 -91 66 1 -91 67 1 -91 68 1 -91 69 1 -91 70 1 -91 71 1 -91 72 1 -91 73 1 -91 74 1 -91 75 1 -91 76 1 -91 77 1 -91 78 1 -91 79 1 -91 80 1 -91 81 1 -91 82 1 -91 83 1 -91 84 1 -91 85 1 -91 86 1 -91 87 1 -91 88 1 -91 89 1 -91 90 1 -91 91 1 -91 92 1 -91 93 1 -91 94 1 -91 95 1 -91 96 1 -91 97 1 -91 98 1 -91 99 1 -91 100 1 -92 0 1 -92 1 1 -92 2 1 -92 3 1 -92 4 1 -92 5 1 -92 6 1 -92 7 1 -92 8 1 -92 9 1 -92 10 1 -92 11 1 -92 12 1 -92 13 1 -92 14 1 -92 15 1 -92 16 1 -92 17 1 -92 18 1 -92 19 1 -92 20 1 -92 21 1 -92 22 1 -92 23 1 -92 24 1 -92 25 1 -92 26 1 -92 27 1 -92 28 1 -92 29 1 -92 30 1 -92 31 1 -92 32 1 -92 33 1 -92 34 1 -92 35 1 -92 36 1 -92 37 1 -92 38 1 -92 39 1 -92 40 1 -92 41 1 -92 42 1 -92 43 1 -92 44 1 -92 45 1 -92 46 1 -92 47 1 -92 48 1 -92 49 1 -92 50 1 -92 51 1 -92 52 1 -92 53 1 -92 54 1 -92 55 1 -92 56 1 -92 57 1 -92 58 1 -92 59 1 -92 60 1 -92 61 1 -92 62 1 -92 63 1 -92 64 1 -92 65 1 -92 66 1 -92 67 1 -92 68 1 -92 69 1 -92 70 1 -92 71 1 -92 72 1 -92 73 1 -92 74 1 -92 75 1 -92 76 1 -92 77 1 -92 78 1 -92 79 1 -92 80 1 -92 81 1 -92 82 1 -92 83 1 -92 84 1 -92 85 1 -92 86 1 -92 87 1 -92 88 1 -92 89 1 -92 90 1 -92 91 1 -92 92 1 -92 93 1 -92 94 1 -92 95 1 -92 96 1 -92 97 1 -92 98 1 -92 99 1 -92 100 1 -93 0 1 -93 1 1 -93 2 1 -93 3 1 -93 4 1 -93 5 1 -93 6 1 -93 7 1 -93 8 1 -93 9 1 -93 10 1 -93 11 1 -93 12 1 -93 13 1 -93 14 1 -93 15 1 -93 16 1 -93 17 1 -93 18 1 -93 19 1 -93 20 1 -93 21 1 -93 22 1 -93 23 1 -93 24 1 -93 25 1 -93 26 1 -93 27 1 -93 28 1 -93 29 1 -93 30 1 -93 31 1 -93 32 1 -93 33 1 -93 34 1 -93 35 1 -93 36 1 -93 37 1 -93 38 1 -93 39 1 -93 40 1 -93 41 1 -93 42 1 -93 43 1 -93 44 1 -93 45 1 -93 46 1 -93 47 1 -93 48 1 -93 49 1 -93 50 1 -93 51 1 -93 52 1 -93 53 1 -93 54 1 -93 55 1 -93 56 1 -93 57 1 -93 58 1 -93 59 1 -93 60 1 -93 61 1 -93 62 1 -93 63 1 -93 64 1 -93 65 1 -93 66 1 -93 67 1 -93 68 1 -93 69 1 -93 70 1 -93 71 1 -93 72 1 -93 73 1 -93 74 1 -93 75 1 -93 76 1 -93 77 1 -93 78 1 -93 79 1 -93 80 1 -93 81 1 -93 82 1 -93 83 1 -93 84 1 -93 85 1 -93 86 1 -93 87 1 -93 88 1 -93 89 1 -93 90 1 -93 91 1 -93 92 1 -93 93 1 -93 94 1 -93 95 1 -93 96 1 -93 97 1 -93 98 1 -93 99 1 -93 100 1 -94 0 1 -94 1 1 -94 2 1 -94 3 1 -94 4 1 -94 5 1 -94 6 1 -94 7 1 -94 8 1 -94 9 1 -94 10 1 -94 11 1 -94 12 1 -94 13 1 -94 14 1 -94 15 1 -94 16 1 -94 17 1 -94 18 1 -94 19 1 -94 20 1 -94 21 1 -94 22 1 -94 23 1 -94 24 1 -94 25 1 -94 26 1 -94 27 1 -94 28 1 -94 29 1 -94 30 1 -94 31 1 -94 32 1 -94 33 1 -94 34 1 -94 35 1 -94 36 1 -94 37 1 -94 38 1 -94 39 1 -94 40 1 -94 41 1 -94 42 1 -94 43 1 -94 44 1 -94 45 1 -94 46 1 -94 47 1 -94 48 1 -94 49 1 -94 50 1 -94 51 1 -94 52 1 -94 53 1 -94 54 1 -94 55 1 -94 56 1 -94 57 1 -94 58 1 -94 59 1 -94 60 1 -94 61 1 -94 62 1 -94 63 1 -94 64 1 -94 65 1 -94 66 1 -94 67 1 -94 68 1 -94 69 1 -94 70 1 -94 71 1 -94 72 1 -94 73 1 -94 74 1 -94 75 1 -94 76 1 -94 77 1 -94 78 1 -94 79 1 -94 80 1 -94 81 1 -94 82 1 -94 83 1 -94 84 1 -94 85 1 -94 86 1 -94 87 1 -94 88 1 -94 89 1 -94 90 1 -94 91 1 -94 92 1 -94 93 1 -94 94 1 -94 95 1 -94 96 1 -94 97 1 -94 98 1 -94 99 1 -94 100 1 -95 0 1 -95 1 1 -95 2 1 -95 3 1 -95 4 1 -95 5 1 -95 6 1 -95 7 1 -95 8 1 -95 9 1 -95 10 1 -95 11 1 -95 12 1 -95 13 1 -95 14 1 -95 15 1 -95 16 1 -95 17 1 -95 18 1 -95 19 1 -95 20 1 -95 21 1 -95 22 1 -95 23 1 -95 24 1 -95 25 1 -95 26 1 -95 27 1 -95 28 1 -95 29 1 -95 30 1 -95 31 1 -95 32 1 -95 33 1 -95 34 1 -95 35 1 -95 36 1 -95 37 1 -95 38 1 -95 39 1 -95 40 1 -95 41 1 -95 42 1 -95 43 1 -95 44 1 -95 45 1 -95 46 1 -95 47 1 -95 48 1 -95 49 1 -95 50 1 -95 51 1 -95 52 1 -95 53 1 -95 54 1 -95 55 1 -95 56 1 -95 57 1 -95 58 1 -95 59 1 -95 60 1 -95 61 1 -95 62 1 -95 63 1 -95 64 1 -95 65 1 -95 66 1 -95 67 1 -95 68 1 -95 69 1 -95 70 1 -95 71 1 -95 72 1 -95 73 1 -95 74 1 -95 75 1 -95 76 1 -95 77 1 -95 78 1 -95 79 1 -95 80 1 -95 81 1 -95 82 1 -95 83 1 -95 84 1 -95 85 1 -95 86 1 -95 87 1 -95 88 1 -95 89 1 -95 90 1 -95 91 1 -95 92 1 -95 93 1 -95 94 1 -95 95 1 -95 96 1 -95 97 1 -95 98 1 -95 99 1 -95 100 1 -96 0 1 -96 1 1 -96 2 1 -96 3 1 -96 4 1 -96 5 1 -96 6 1 -96 7 1 -96 8 1 -96 9 1 -96 10 1 -96 11 1 -96 12 1 -96 13 1 -96 14 1 -96 15 1 -96 16 1 -96 17 1 -96 18 1 -96 19 1 -96 20 1 -96 21 1 -96 22 1 -96 23 1 -96 24 1 -96 25 1 -96 26 1 -96 27 1 -96 28 1 -96 29 1 -96 30 1 -96 31 1 -96 32 1 -96 33 1 -96 34 1 -96 35 1 -96 36 1 -96 37 1 -96 38 1 -96 39 1 -96 40 1 -96 41 1 -96 42 1 -96 43 1 -96 44 1 -96 45 1 -96 46 1 -96 47 1 -96 48 1 -96 49 1 -96 50 1 -96 51 1 -96 52 1 -96 53 1 -96 54 1 -96 55 1 -96 56 1 -96 57 1 -96 58 1 -96 59 1 -96 60 1 -96 61 1 -96 62 1 -96 63 1 -96 64 1 -96 65 1 -96 66 1 -96 67 1 -96 68 1 -96 69 1 -96 70 1 -96 71 1 -96 72 1 -96 73 1 -96 74 1 -96 75 1 -96 76 1 -96 77 1 -96 78 1 -96 79 1 -96 80 1 -96 81 1 -96 82 1 -96 83 1 -96 84 1 -96 85 1 -96 86 1 -96 87 1 -96 88 1 -96 89 1 -96 90 1 -96 91 1 -96 92 1 -96 93 1 -96 94 1 -96 95 1 -96 96 1 -96 97 1 -96 98 1 -96 99 1 -96 100 1 -97 0 1 -97 1 1 -97 2 1 -97 3 1 -97 4 1 -97 5 1 -97 6 1 -97 7 1 -97 8 1 -97 9 1 -97 10 1 -97 11 1 -97 12 1 -97 13 1 -97 14 1 -97 15 1 -97 16 1 -97 17 1 -97 18 1 -97 19 1 -97 20 1 -97 21 1 -97 22 1 -97 23 1 -97 24 1 -97 25 1 -97 26 1 -97 27 1 -97 28 1 -97 29 1 -97 30 1 -97 31 1 -97 32 1 -97 33 1 -97 34 1 -97 35 1 -97 36 1 -97 37 1 -97 38 1 -97 39 1 -97 40 1 -97 41 1 -97 42 1 -97 43 1 -97 44 1 -97 45 1 -97 46 1 -97 47 1 -97 48 1 -97 49 1 -97 50 1 -97 51 1 -97 52 1 -97 53 1 -97 54 1 -97 55 1 -97 56 1 -97 57 1 -97 58 1 -97 59 1 -97 60 1 -97 61 1 -97 62 1 -97 63 1 -97 64 1 -97 65 1 -97 66 1 -97 67 1 -97 68 1 -97 69 1 -97 70 1 -97 71 1 -97 72 1 -97 73 1 -97 74 1 -97 75 1 -97 76 1 -97 77 1 -97 78 1 -97 79 1 -97 80 1 -97 81 1 -97 82 1 -97 83 1 -97 84 1 -97 85 1 -97 86 1 -97 87 1 -97 88 1 -97 89 1 -97 90 1 -97 91 1 -97 92 1 -97 93 1 -97 94 1 -97 95 1 -97 96 1 -97 97 1 -97 98 1 -97 99 1 -97 100 1 -98 0 1 -98 1 1 -98 2 1 -98 3 1 -98 4 1 -98 5 1 -98 6 1 -98 7 1 -98 8 1 -98 9 1 -98 10 1 -98 11 1 -98 12 1 -98 13 1 -98 14 1 -98 15 1 -98 16 1 -98 17 1 -98 18 1 -98 19 1 -98 20 1 -98 21 1 -98 22 1 -98 23 1 -98 24 1 -98 25 1 -98 26 1 -98 27 1 -98 28 1 -98 29 1 -98 30 1 -98 31 1 -98 32 1 -98 33 1 -98 34 1 -98 35 1 -98 36 1 -98 37 1 -98 38 1 -98 39 1 -98 40 1 -98 41 1 -98 42 1 -98 43 1 -98 44 1 -98 45 1 -98 46 1 -98 47 1 -98 48 1 -98 49 1 -98 50 1 -98 51 1 -98 52 1 -98 53 1 -98 54 1 -98 55 1 -98 56 1 -98 57 1 -98 58 1 -98 59 1 -98 60 1 -98 61 1 -98 62 1 -98 63 1 -98 64 1 -98 65 1 -98 66 1 -98 67 1 -98 68 1 -98 69 1 -98 70 1 -98 71 1 -98 72 1 -98 73 1 -98 74 1 -98 75 1 -98 76 1 -98 77 1 -98 78 1 -98 79 1 -98 80 1 -98 81 1 -98 82 1 -98 83 1 -98 84 1 -98 85 1 -98 86 1 -98 87 1 -98 88 1 -98 89 1 -98 90 1 -98 91 1 -98 92 1 -98 93 1 -98 94 1 -98 95 1 -98 96 1 -98 97 1 -98 98 1 -98 99 1 -98 100 1 -99 0 1 -99 1 1 -99 2 1 -99 3 1 -99 4 1 -99 5 1 -99 6 1 -99 7 1 -99 8 1 -99 9 1 -99 10 1 -99 11 1 -99 12 1 -99 13 1 -99 14 1 -99 15 1 -99 16 1 -99 17 1 -99 18 1 -99 19 1 -99 20 1 -99 21 1 -99 22 1 -99 23 1 -99 24 1 -99 25 1 -99 26 1 -99 27 1 -99 28 1 -99 29 1 -99 30 1 -99 31 1 -99 32 1 -99 33 1 -99 34 1 -99 35 1 -99 36 1 -99 37 1 -99 38 1 -99 39 1 -99 40 1 -99 41 1 -99 42 1 -99 43 1 -99 44 1 -99 45 1 -99 46 1 -99 47 1 -99 48 1 -99 49 1 -99 50 1 -99 51 1 -99 52 1 -99 53 1 -99 54 1 -99 55 1 -99 56 1 -99 57 1 -99 58 1 -99 59 1 -99 60 1 -99 61 1 -99 62 1 -99 63 1 -99 64 1 -99 65 1 -99 66 1 -99 67 1 -99 68 1 -99 69 1 -99 70 1 -99 71 1 -99 72 1 -99 73 1 -99 74 1 -99 75 1 -99 76 1 -99 77 1 -99 78 1 -99 79 1 -99 80 1 -99 81 1 -99 82 1 -99 83 1 -99 84 1 -99 85 1 -99 86 1 -99 87 1 -99 88 1 -99 89 1 -99 90 1 -99 91 1 -99 92 1 -99 93 1 -99 94 1 -99 95 1 -99 96 1 -99 97 1 -99 98 1 -99 99 1 -99 100 1 -100 0 1 -100 1 1 -100 2 1 -100 3 1 -100 4 1 -100 5 1 -100 6 1 -100 7 1 -100 8 1 -100 9 1 -100 10 1 -100 11 1 -100 12 1 -100 13 1 -100 14 1 -100 15 1 -100 16 1 -100 17 1 -100 18 1 -100 19 1 -100 20 1 -100 21 1 -100 22 1 -100 23 1 -100 24 1 -100 25 1 -100 26 1 -100 27 1 -100 28 1 -100 29 1 -100 30 1 -100 31 1 -100 32 1 -100 33 1 -100 34 1 -100 35 1 -100 36 1 -100 37 1 -100 38 1 -100 39 1 -100 40 1 -100 41 1 -100 42 1 -100 43 1 -100 44 1 -100 45 1 -100 46 1 -100 47 1 -100 48 1 -100 49 1 -100 50 1 -100 51 1 -100 52 1 -100 53 1 -100 54 1 -100 55 1 -100 56 1 -100 57 1 -100 58 1 -100 59 1 -100 60 1 -100 61 1 -100 62 1 -100 63 1 -100 64 1 -100 65 1 -100 66 1 -100 67 1 -100 68 1 -100 69 1 -100 70 1 -100 71 1 -100 72 1 -100 73 1 -100 74 1 -100 75 1 -100 76 1 -100 77 1 -100 78 1 -100 79 1 -100 80 1 -100 81 1 -100 82 1 -100 83 1 -100 84 1 -100 85 1 -100 86 1 -100 87 1 -100 88 1 -100 89 1 -100 90 1 -100 91 1 -100 92 1 -100 93 1 -100 94 1 -100 95 1 -100 96 1 -100 97 1 -100 98 1 -100 99 1 -100 100 1 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.srcsegmlentable b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.srcsegmlentable deleted file mode 100644 index 44ee42e20..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.srcsegmlentable +++ /dev/null @@ -1 +0,0 @@ -Uniform diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.trgsegmlentable b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.trgsegmlentable deleted file mode 100644 index 80dd323ca..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.trgsegmlentable +++ /dev/null @@ -1 +0,0 @@ -Geometric diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.ttable b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg.ttable deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.anji b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.anji deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.anjm1ip_anji b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.anjm1ip_anji deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.asifactor b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.asifactor deleted file mode 100644 index be5863417..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.asifactor +++ /dev/null @@ -1 +0,0 @@ -0.3 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.hmm_alignd b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.hmm_alignd deleted file mode 100644 index d17ce6ae62383450a8678872b637c8acf7c6307a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22 VcmZQzU|?W`VqRXZ>&kmTBmfN}0@wfm diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.hmm_lexnd b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.hmm_lexnd deleted file mode 100644 index 67392c2b8f0fccb590a0bc1afd6e029d3d68be3d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32 ccmZQzU|?Vd;t%Hz?gwEI8-!Qn)a?Ud0GiSZCIA2c diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.hmm_p0 b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.hmm_p0 deleted file mode 100644 index 49d59571f..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.hmm_p0 +++ /dev/null @@ -1 +0,0 @@ -0.1 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.lsifactor b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.lsifactor deleted file mode 100644 index 49d59571f..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.lsifactor +++ /dev/null @@ -1 +0,0 @@ -0.1 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.msinfo b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.msinfo deleted file mode 100644 index aa47d0d46..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.msinfo +++ /dev/null @@ -1,2 +0,0 @@ -0 -0 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.slmodel b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.slmodel deleted file mode 100644 index 1e5ffcdb9..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.slmodel +++ /dev/null @@ -1,3 +0,0 @@ -Weighted incr. gaussian sentence length model... -numsents: 1 ; slensum: 1 ; tlensum: 1 -1 1 1.00000000 1.00000000 0.00000000 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.src b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.src deleted file mode 100644 index 8b1378917..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.src +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.srctrgc b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.srctrgc deleted file mode 100644 index d00491fd7..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.srctrgc +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.svcb b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.svcb deleted file mode 100644 index e04006a57..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.svcb +++ /dev/null @@ -1,4 +0,0 @@ -2 0 -3 1 -0 NULL 0 -1 UNKNOWN_WORD 0 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.trg b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.trg deleted file mode 100644 index 8b1378917..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.trg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.tvcb b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.tvcb deleted file mode 100644 index e04006a57..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_invswm.tvcb +++ /dev/null @@ -1,4 +0,0 @@ -2 0 -3 1 -0 NULL 0 -1 UNKNOWN_WORD 0 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.anji b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.anji deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.anjm1ip_anji b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.anjm1ip_anji deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.asifactor b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.asifactor deleted file mode 100644 index be5863417..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.asifactor +++ /dev/null @@ -1 +0,0 @@ -0.3 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.hmm_alignd b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.hmm_alignd deleted file mode 100644 index d17ce6ae62383450a8678872b637c8acf7c6307a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22 VcmZQzU|?W`VqRXZ>&kmTBmfN}0@wfm diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.hmm_lexnd b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.hmm_lexnd deleted file mode 100644 index 67392c2b8f0fccb590a0bc1afd6e029d3d68be3d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 32 ccmZQzU|?Vd;t%Hz?gwEI8-!Qn)a?Ud0GiSZCIA2c diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.hmm_p0 b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.hmm_p0 deleted file mode 100644 index 49d59571f..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.hmm_p0 +++ /dev/null @@ -1 +0,0 @@ -0.1 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.lsifactor b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.lsifactor deleted file mode 100644 index 49d59571f..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.lsifactor +++ /dev/null @@ -1 +0,0 @@ -0.1 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.msinfo b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.msinfo deleted file mode 100644 index aa47d0d46..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.msinfo +++ /dev/null @@ -1,2 +0,0 @@ -0 -0 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.slmodel b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.slmodel deleted file mode 100644 index 1e5ffcdb9..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.slmodel +++ /dev/null @@ -1,3 +0,0 @@ -Weighted incr. gaussian sentence length model... -numsents: 1 ; slensum: 1 ; tlensum: 1 -1 1 1.00000000 1.00000000 0.00000000 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.src b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.src deleted file mode 100644 index 8b1378917..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.src +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.srctrgc b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.srctrgc deleted file mode 100644 index d00491fd7..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.srctrgc +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.svcb b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.svcb deleted file mode 100644 index e04006a57..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.svcb +++ /dev/null @@ -1,4 +0,0 @@ -2 0 -3 1 -0 NULL 0 -1 UNKNOWN_WORD 0 diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.trg b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.trg deleted file mode 100644 index 8b1378917..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.trg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.tvcb b/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.tvcb deleted file mode 100644 index e04006a57..000000000 --- a/src/SIL.Machine.AspNetCore/data/thot-new-model/tm/src_trg_swm.tvcb +++ /dev/null @@ -1,4 +0,0 @@ -2 0 -3 1 -0 NULL 0 -1 UNKNOWN_WORD 0 diff --git a/src/SIL.Machine.Serval.EngineServer/Program.cs b/src/SIL.Machine.Serval.EngineServer/Program.cs deleted file mode 100644 index 029e03df9..000000000 --- a/src/SIL.Machine.Serval.EngineServer/Program.cs +++ /dev/null @@ -1,39 +0,0 @@ -using Hangfire; -using OpenTelemetry.Trace; - -var builder = WebApplication.CreateBuilder(args); - -// Add services to the container. -builder - .Services.AddMachine(builder.Configuration) - .AddBuildJobService() - .AddMongoDataAccess() - .AddMongoHangfireJobClient() - .AddServalTranslationEngineService() - .AddModelCleanupService() - .AddMessageOutboxDeliveryService() - .AddClearMLService(); - -if (builder.Environment.IsDevelopment()) -{ - builder - .Services.AddOpenTelemetry() - .WithTracing(builder => - { - builder - .AddAspNetCoreInstrumentation() - .AddHttpClientInstrumentation() - .AddGrpcClientInstrumentation() - .AddSource("MongoDB.Driver.Core.Extensions.DiagnosticSources") - .AddConsoleExporter(); - }); -} - -var app = builder.Build(); - -app.UseHttpsRedirection(); - -app.MapServalTranslationEngineService(); -app.MapHangfireDashboard(); - -app.Run(); diff --git a/src/SIL.Machine.Serval.EngineServer/Properties/launchSettings.json b/src/SIL.Machine.Serval.EngineServer/Properties/launchSettings.json deleted file mode 100644 index 34eb2e94b..000000000 --- a/src/SIL.Machine.Serval.EngineServer/Properties/launchSettings.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "profiles": { - "SIL.Machine.Serval.EngineServer": { - "commandName": "Project", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - }, - "applicationUrl": "https://localhost:9000" - } - } -} \ No newline at end of file diff --git a/src/SIL.Machine.Serval.EngineServer/SIL.Machine.Serval.EngineServer.csproj b/src/SIL.Machine.Serval.EngineServer/SIL.Machine.Serval.EngineServer.csproj deleted file mode 100644 index 46f70dd98..000000000 --- a/src/SIL.Machine.Serval.EngineServer/SIL.Machine.Serval.EngineServer.csproj +++ /dev/null @@ -1,36 +0,0 @@ - - - - net8.0 - enable - enable - 34e222a9-ef76-48f9-869e-338547f9bd25 - true - true - true - $(NoWarn);CS1591;CS1573 - - - - - - - - - - - - - - - - - - - - - icu.net.dll.config - - - - diff --git a/src/SIL.Machine.Serval.EngineServer/appsettings.Development.json b/src/SIL.Machine.Serval.EngineServer/appsettings.Development.json deleted file mode 100644 index 1f2a4ef6b..000000000 --- a/src/SIL.Machine.Serval.EngineServer/appsettings.Development.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "ConnectionStrings": { - "Hangfire": "mongodb://localhost:27017/machine_jobs", - "Mongo": "mongodb://localhost:27017/machine", - "Serval": "https://localhost:8444" - }, - "ClearML": { - "MaxSteps": 1000, - "Project": "dev" - }, - "SharedFile": { - "Uri": "s3://aqua-ml-data/dev/" - }, - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning", - "System.Net.Http.HttpClient.Default": "Warning" - } - } -} \ No newline at end of file diff --git a/src/SIL.Machine.Serval.EngineServer/appsettings.Production.json b/src/SIL.Machine.Serval.EngineServer/appsettings.Production.json deleted file mode 100644 index 1b2d3bafd..000000000 --- a/src/SIL.Machine.Serval.EngineServer/appsettings.Production.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - } -} \ No newline at end of file diff --git a/src/SIL.Machine.Serval.EngineServer/appsettings.Staging.json b/src/SIL.Machine.Serval.EngineServer/appsettings.Staging.json deleted file mode 100644 index 1b2d3bafd..000000000 --- a/src/SIL.Machine.Serval.EngineServer/appsettings.Staging.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - } -} \ No newline at end of file diff --git a/src/SIL.Machine.Serval.EngineServer/appsettings.json b/src/SIL.Machine.Serval.EngineServer/appsettings.json deleted file mode 100644 index 271163fff..000000000 --- a/src/SIL.Machine.Serval.EngineServer/appsettings.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "ConnectionStrings": { - "ClearML": "https://api.sil.hosted.allegro.ai" - }, - "AllowedHosts": "*", - "Service": { - "ServiceId": "machine_engine" - }, - "TranslationEngines": [ - "SmtTransfer", - "Nmt" - ], - "BuildJob": { - "ClearML": [ - { - "TranslationEngineType": "Nmt", - "ModelType": "huggingface", - "Queue": "jobs_backlog", - "DockerImage": "ghcr.io/sillsdev/machine.py:latest" - }, - { - "TranslationEngineType": "SmtTransfer", - "ModelType": "thot", - "Queue": "cpu_only", - "DockerImage": "ghcr.io/sillsdev/machine.py:latest" - } - ] - }, - "SmtTransferEngine": { - "EnginesDir": "/var/lib/machine/engines" - }, - "ClearML": { - "BuildPollingEnabled": true - }, - "MessageOutbox": { - "OutboxDir": "/var/lib/machine/outbox" - }, - "Logging": { - "LogLevel": { - "System.Net.Http.HttpClient.Default": "Warning" - } - } -} \ No newline at end of file diff --git a/src/SIL.Machine.Serval.JobServer/Program.cs b/src/SIL.Machine.Serval.JobServer/Program.cs deleted file mode 100644 index d78bfed80..000000000 --- a/src/SIL.Machine.Serval.JobServer/Program.cs +++ /dev/null @@ -1,30 +0,0 @@ -using OpenTelemetry.Trace; - -var builder = WebApplication.CreateBuilder(args); - -builder - .Services.AddMachine(builder.Configuration) - .AddBuildJobService() - .AddMongoDataAccess() - .AddMongoHangfireJobClient() - .AddHangfireJobServer() - .AddServalPlatformService() - .AddClearMLService(); -if (builder.Environment.IsDevelopment()) -{ - builder - .Services.AddOpenTelemetry() - .WithTracing(builder => - { - builder - .AddAspNetCoreInstrumentation() - .AddHttpClientInstrumentation() - .AddGrpcClientInstrumentation() - .AddSource("MongoDB.Driver.Core.Extensions.DiagnosticSources") - .AddConsoleExporter(); - }); -} - -var app = builder.Build(); - -app.Run(); diff --git a/src/SIL.Machine.Serval.JobServer/Properties/launchSettings.json b/src/SIL.Machine.Serval.JobServer/Properties/launchSettings.json deleted file mode 100644 index f636d0c39..000000000 --- a/src/SIL.Machine.Serval.JobServer/Properties/launchSettings.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "profiles": { - "SIL.Machine.Serval.JobServer": { - "commandName": "Project", - "launchBrowser": false, - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - }, - "applicationUrl": "https://localhost:9100" - } - } -} \ No newline at end of file diff --git a/src/SIL.Machine.Serval.JobServer/SIL.Machine.Serval.JobServer.csproj b/src/SIL.Machine.Serval.JobServer/SIL.Machine.Serval.JobServer.csproj deleted file mode 100644 index 983f92516..000000000 --- a/src/SIL.Machine.Serval.JobServer/SIL.Machine.Serval.JobServer.csproj +++ /dev/null @@ -1,37 +0,0 @@ - - - - net8.0 - enable - enable - aa9e7440-5a04-4de6-ba51-bab9ef4a62e1 - true - true - true - $(NoWarn);CS1591;CS1573 - - - - - - - - - - - - - - - - - - - - - - icu.net.dll.config - - - - diff --git a/src/SIL.Machine.Serval.JobServer/appsettings.Development.json b/src/SIL.Machine.Serval.JobServer/appsettings.Development.json deleted file mode 100644 index 1f2a4ef6b..000000000 --- a/src/SIL.Machine.Serval.JobServer/appsettings.Development.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "ConnectionStrings": { - "Hangfire": "mongodb://localhost:27017/machine_jobs", - "Mongo": "mongodb://localhost:27017/machine", - "Serval": "https://localhost:8444" - }, - "ClearML": { - "MaxSteps": 1000, - "Project": "dev" - }, - "SharedFile": { - "Uri": "s3://aqua-ml-data/dev/" - }, - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning", - "System.Net.Http.HttpClient.Default": "Warning" - } - } -} \ No newline at end of file diff --git a/src/SIL.Machine.Serval.JobServer/appsettings.Production.json b/src/SIL.Machine.Serval.JobServer/appsettings.Production.json deleted file mode 100644 index 1b2d3bafd..000000000 --- a/src/SIL.Machine.Serval.JobServer/appsettings.Production.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - } -} \ No newline at end of file diff --git a/src/SIL.Machine.Serval.JobServer/appsettings.Staging.json b/src/SIL.Machine.Serval.JobServer/appsettings.Staging.json deleted file mode 100644 index 1b2d3bafd..000000000 --- a/src/SIL.Machine.Serval.JobServer/appsettings.Staging.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - } -} \ No newline at end of file diff --git a/src/SIL.Machine.Serval.JobServer/appsettings.json b/src/SIL.Machine.Serval.JobServer/appsettings.json deleted file mode 100644 index 738a4f288..000000000 --- a/src/SIL.Machine.Serval.JobServer/appsettings.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "ConnectionStrings": { - "ClearML": "https://api.sil.hosted.allegro.ai" - }, - "AllowedHosts": "*", - "Service": { - "ServiceId": "machine_job" - }, - "TranslationEngines": [ - "SmtTransfer", - "Nmt" - ], - "BuildJob": { - "ClearML": [ - { - "TranslationEngineType": "Nmt", - "ModelType": "huggingface", - "Queue": "jobs_backlog", - "DockerImage": "ghcr.io/sillsdev/machine.py:latest" - }, - { - "TranslationEngineType": "SmtTransfer", - "ModelType": "thot", - "Queue": "jobs_backlog", - "DockerImage": "ghcr.io/sillsdev/machine.py:latest" - } - ] - }, - "SmtTransferEngine": { - "EnginesDir": "/var/lib/machine/engines" - }, - "ClearML": { - "BuildPollingEnabled": false - }, - "MessageOutbox": { - "OutboxDir": "/var/lib/machine/outbox" - }, - "Logging": { - "LogLevel": { - "System.Net.Http.HttpClient.Default": "Warning" - } - } -} \ No newline at end of file diff --git a/tests/SIL.Machine.AspNetCore.Tests/SIL.Machine.AspNetCore.Tests.csproj b/tests/SIL.Machine.AspNetCore.Tests/SIL.Machine.AspNetCore.Tests.csproj deleted file mode 100644 index f9f221847..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/SIL.Machine.AspNetCore.Tests.csproj +++ /dev/null @@ -1,52 +0,0 @@ - - - - net8.0 - SIL.Machine.AspNetCore - enable - enable - false - true - true - true - $(NoWarn);CS1591;CS1573 - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - - diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLServiceTests.cs deleted file mode 100644 index e4a9f75f8..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/ClearMLServiceTests.cs +++ /dev/null @@ -1,50 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class ClearMLServiceTests -{ - private const string ApiServer = "https://clearml.com"; - private const string AccessKey = "accessKey"; - private const string SecretKey = "secretKey"; - - [Test] - public async Task CreateTaskAsync() - { - var mockHttp = new MockHttpMessageHandler(); - mockHttp - .Expect(HttpMethod.Post, $"{ApiServer}/tasks.create") - .WithHeaders("Authorization", $"Bearer accessToken") - .WithPartialContent("\\u0027src_lang\\u0027: \\u0027spa_Latn\\u0027") - .WithPartialContent("\\u0027trg_lang\\u0027: \\u0027eng_Latn\\u0027") - .Respond("application/json", "{ \"data\": { \"id\": \"projectId\" } }"); - HttpClient httpClient = mockHttp.ToHttpClient(); - httpClient.BaseAddress = new Uri(ApiServer); - - var options = Substitute.For>(); - options.CurrentValue.Returns(new ClearMLOptions { AccessKey = AccessKey, SecretKey = SecretKey }); - var authService = Substitute.For(); - authService.GetAuthTokenAsync().Returns(Task.FromResult("accessToken")); - var env = new HostingEnvironment { EnvironmentName = Environments.Development }; - var httpClientFactory = Substitute.For(); - httpClientFactory.CreateClient("ClearML").Returns(httpClient); - var service = new ClearMLService(httpClientFactory, options, authService, env); - - string script = - "from machine.jobs.build_nmt_engine import run\n" - + "args = {\n" - + " 'model_type': 'huggingface',\n" - + " 'engine_id': 'engine1',\n" - + " 'build_id': 'build1',\n" - + " 'src_lang': 'spa_Latn',\n" - + " 'trg_lang': 'eng_Latn',\n" - + " 'max_steps': 20000,\n" - + " 'shared_file_uri': 's3://aqua-ml-data',\n" - + " 'clearml': True\n" - + "}\n" - + "run(args)\n"; - - string projectId = await service.CreateTaskAsync("build1", "project1", "dockerImage", script); - Assert.That(projectId, Is.EqualTo("projectId")); - mockHttp.VerifyNoOutstandingExpectation(); - } -} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/DistributedReaderWriterLockFactoryTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/DistributedReaderWriterLockFactoryTests.cs deleted file mode 100644 index 61a5cbfd2..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/DistributedReaderWriterLockFactoryTests.cs +++ /dev/null @@ -1,81 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class DistributedReaderWriterLockFactoryTests -{ - [Test] - public async Task InitAsync_ReleaseWriterLocks() - { - TestEnvironment env = new(); - env.Locks.Add( - new RWLock - { - Id = "resource1", - WriterLock = new() { Id = "lock1", HostId = "this_service" }, - ReaderLocks = [], - WriterQueue = [] - } - ); - - await env.Factory.InitAsync(); - - RWLock resource1 = env.Locks.Get("resource1"); - Assert.That(resource1.WriterLock, Is.Null); - } - - [Test] - public async Task InitAsync_ReleaseReaderLocks() - { - TestEnvironment env = new(); - env.Locks.Add( - new RWLock - { - Id = "resource1", - ReaderLocks = [new() { Id = "lock1", HostId = "this_service" }], - WriterQueue = [] - } - ); - - await env.Factory.InitAsync(); - - RWLock resource1 = env.Locks.Get("resource1"); - Assert.That(resource1.ReaderLocks, Is.Empty); - } - - [Test] - public async Task InitAsync_RemoveWaiters() - { - TestEnvironment env = new(); - env.Locks.Add( - new RWLock - { - Id = "resource1", - WriterLock = new() { Id = "lock1", HostId = "other_service" }, - ReaderLocks = [], - WriterQueue = [new() { Id = "lock2", HostId = "this_service" }] - } - ); - - await env.Factory.InitAsync(); - - RWLock resource1 = env.Locks.Get("resource1"); - Assert.That(resource1.WriterQueue, Is.Empty); - } - - private class TestEnvironment - { - public TestEnvironment() - { - Locks = new MemoryRepository(); - ServiceOptions serviceOptions = new() { ServiceId = "this_service" }; - Factory = new DistributedReaderWriterLockFactory( - new OptionsWrapper(serviceOptions), - Locks, - new ObjectIdGenerator() - ); - } - - public MemoryRepository Locks { get; } - public DistributedReaderWriterLockFactory Factory { get; } - } -} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/DistributedReaderWriterLockTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/DistributedReaderWriterLockTests.cs deleted file mode 100644 index ac08d2f85..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/DistributedReaderWriterLockTests.cs +++ /dev/null @@ -1,405 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class DistributedReaderWriterLockTests -{ - [Test] - public async Task ReaderLockAsync_NoLockAcquired() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - RWLock entity; - await using (await rwLock.ReaderLockAsync()) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task ReaderLockAsync_ReaderLockAcquired() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - RWLock entity; - await using (await rwLock.ReaderLockAsync()) - { - await using (await rwLock.ReaderLockAsync()) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task ReaderLockAsync_WriterLockAcquiredAndNotReleased() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - await rwLock.WriterLockAsync(); - var task = rwLock.ReaderLockAsync(); - await AssertNeverCompletesAsync(task); - } - - [Test] - public async Task ReaderLockAsync_WriterLockAcquiredAndReleased() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - Task task; - await using (await rwLock.WriterLockAsync()) - { - task = rwLock.ReaderLockAsync(); - Assert.That(task.IsCompleted, Is.False); - } - - RWLock entity; - await using (await task) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task ReaderLockAsync_WriterLockAcquiredAndExpired() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - RWLock entity; - await using (await rwLock.WriterLockAsync(TimeSpan.FromMilliseconds(400))) - { - var task = rwLock.ReaderLockAsync(); - await Task.Delay(500); - await using (await task) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task ReaderLockAsync_Cancelled() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - Task task; - await using (await rwLock.WriterLockAsync()) - { - var cts = new CancellationTokenSource(); - task = rwLock.ReaderLockAsync(cancellationToken: cts.Token); - cts.Cancel(); - Assert.CatchAsync(async () => await task); - } - - RWLock entity; - await using (await rwLock.ReaderLockAsync()) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task WriterLockAsync_NoLockAcquired() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - RWLock entity; - await using (await rwLock.WriterLockAsync()) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.False); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task WriterLockAsync_ReaderLockAcquiredAndNotReleased() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - await rwLock.ReaderLockAsync(); - var task = rwLock.WriterLockAsync(); - await AssertNeverCompletesAsync(task); - } - - [Test] - public async Task WriterLockAsync_ReaderLockAcquiredAndReleased() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - Task task; - await using (await rwLock.ReaderLockAsync()) - { - task = rwLock.WriterLockAsync(); - Assert.That(task.IsCompleted, Is.False); - } - - RWLock entity; - await using (await task) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.False); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task WriterLockAsync_WriterLockAcquiredAndNeverReleased() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - await rwLock.WriterLockAsync(); - var task = rwLock.WriterLockAsync(); - await AssertNeverCompletesAsync(task); - } - - [Test] - public async Task WriterLockAsync_WriterLockAcquiredAndReleased() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - Task task; - await using (await rwLock.WriterLockAsync()) - { - task = rwLock.WriterLockAsync(); - Assert.That(task.IsCompleted, Is.False); - } - - RWLock entity; - await using (await task) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.False); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task WriterLockAsync_WriterLockTakesPriorityOverReaderLock() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - Task writeTask, - readTask; - await using (await rwLock.WriterLockAsync()) - { - readTask = rwLock.ReaderLockAsync(); - Assert.That(readTask.IsCompleted, Is.False); - writeTask = rwLock.WriterLockAsync(); - Assert.That(writeTask.IsCompleted, Is.False); - } - - await writeTask; - await AssertNeverCompletesAsync(readTask); - } - - [Test] - public async Task WriterLockAsync_FirstWriterLockHasPriority() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - Task task1, - task2; - await using (await rwLock.WriterLockAsync()) - { - task1 = rwLock.WriterLockAsync(); - Assert.That(task1.IsCompleted, Is.False); - task2 = rwLock.WriterLockAsync(); - Assert.That(task2.IsCompleted, Is.False); - } - - await task1; - await AssertNeverCompletesAsync(task2); - } - - [Test] - public async Task WriterLockAsync_WriterLockAcquiredAndExpired() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - RWLock entity; - await using (await rwLock.WriterLockAsync(TimeSpan.FromMilliseconds(400))) - { - var task = rwLock.WriterLockAsync(); - await Task.Delay(500); - await using (await task) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.False); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - [Test] - public async Task WriterLockAsync_Cancelled() - { - var env = new TestEnvironment(); - IDistributedReaderWriterLock rwLock = await env.Factory.CreateAsync("test"); - - Task task; - await using (await rwLock.WriterLockAsync()) - { - var cts = new CancellationTokenSource(); - task = rwLock.WriterLockAsync(cancellationToken: cts.Token); - cts.Cancel(); - Assert.CatchAsync(async () => await task); - } - - RWLock entity; - await using (await rwLock.WriterLockAsync()) - { - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.False); - Assert.That(entity.IsAvailableForWriting(), Is.False); - }); - } - - entity = env.Locks.Get("test"); - Assert.Multiple(() => - { - Assert.That(entity.IsAvailableForReading(), Is.True); - Assert.That(entity.IsAvailableForWriting(), Is.True); - }); - } - - private static async Task AssertNeverCompletesAsync(Task task, int timeout = 100) - { - if (task.IsCompleted) - Assert.Fail("Task completed unexpectedly."); - Task completedTask = await Task.WhenAny(task, Task.Delay(timeout)).ConfigureAwait(false); - if (completedTask == task) - Assert.Fail("Task completed unexpectedly."); - var _ = task.ContinueWith(_ => Assert.Fail("Task completed unexpectedly."), TaskScheduler.Default); - } - - private class TestEnvironment - { - public TestEnvironment() - { - Locks = new MemoryRepository(); - var idGenerator = new ObjectIdGenerator(); - var options = Substitute.For>(); - options.Value.Returns(new ServiceOptions { ServiceId = "host" }); - Factory = new DistributedReaderWriterLockFactory(options, Locks, idGenerator); - } - - public DistributedReaderWriterLockFactory Factory { get; } - public MemoryRepository Locks { get; } - } -} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/InMemoryStorageTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/InMemoryStorageTests.cs deleted file mode 100644 index 3b5052865..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/InMemoryStorageTests.cs +++ /dev/null @@ -1,91 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class InMemoryStorageTests -{ - [Test] - public async Task ExistsAsync() - { - using InMemoryStorage fs = new(); - using (StreamWriter sw = new(await fs.OpenWriteAsync("file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - bool exists = await fs.ExistsAsync("file1"); - Assert.That(exists, Is.True); - } - - [Test] - public async Task OpenReadAsync() - { - using InMemoryStorage fs = new(); - string input; - using (StreamWriter sw = new(await fs.OpenWriteAsync("file1"))) - { - input = "Hello"; - sw.WriteLine(input); - } - string? output; - using (StreamReader sr = new(await fs.OpenReadAsync("file1"))) - { - output = sr.ReadLine(); - } - Assert.That(input, Is.EqualTo(output), $"{input} | {output}"); - } - - [Test] - public async Task ListFilesAsync_Recurse() - { - using InMemoryStorage fs = new(); - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) - { - string input2 = "Hola"; - sw.WriteLine(input2); - } - IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: true); - Assert.That(files, Is.EquivalentTo(new[] { "test/file1", "test/test/file2" })); - } - - [Test] - public async Task ListFilesAsync_DoNotRecurse() - { - using InMemoryStorage fs = new(); - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) - { - string input2 = "Hola"; - sw.WriteLine(input2); - } - IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: false); - Assert.That(files, Is.EquivalentTo(new[] { "test/file1" })); - } - - [Test] - public async Task DeleteAsync() - { - using InMemoryStorage fs = new(); - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) - { - string input2 = "Hola"; - sw.WriteLine(input2); - } - await fs.DeleteAsync("test", recurse: true); - var files = await fs.ListFilesAsync("test", recurse: true); - Assert.That(files, Is.Empty); - } -} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/LanguageTagServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/LanguageTagServiceTests.cs deleted file mode 100644 index 593ba8524..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/LanguageTagServiceTests.cs +++ /dev/null @@ -1,69 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class LanguageTagServiceTests -{ - [Test] - [TestCase("es", "spa_Latn", Description = "Iso639_1Code")] - [TestCase("hne", "hne_Deva", Description = "Iso639_3Code")] - [TestCase("ks-Arab", "kas_Arab", Description = "ScriptCode")] - [TestCase("srp_Cyrl", "srp_Cyrl", Description = "InvalidLangTag")] - [TestCase("zh", "zho_Hans", Description = "ChineseNoScript")] - [TestCase("zh-Hant", "zho_Hant", Description = "ChineseScript")] - [TestCase("zh-TW", "zho_Hant", Description = "ChineseRegion")] - [TestCase("cmn", "zho_Hans", Description = "MandarinChineseNoScript")] - [TestCase("cmn-Hant", "zho_Hant", Description = "MandarinChineseScript")] - [TestCase("ms", "zsm_Latn", Description = "Macrolanguage")] - [TestCase("arb", "arb_Arab", Description = "Arabic")] - [TestCase("eng", "eng_Latn", Description = "InsteadOfISO639_1")] - [TestCase("eng-Latn", "eng_Latn", Description = "DashToUnderscore")] - [TestCase("kor", "kor_Hang", Description = "KoreanScript")] - [TestCase("kor_Kore", "kor_Hang", Description = "KoreanScriptCorrection")] - public void ConvertToFlores200CodeTest(string language, string internalCodeTruth) - { - if (!Sldr.IsInitialized) - Sldr.Initialize(); - new LanguageTagService().ConvertToFlores200Code(language, out string internalCode); - Assert.That(internalCode, Is.EqualTo(internalCodeTruth)); - } - - [Test] - [TestCase("en", "eng_Latn", true)] - [TestCase("ms", "zsm_Latn", true)] - [TestCase("cmn", "zho_Hans", true)] - [TestCase("xyz", "xyz", false)] - public void GetLanguageInfoAsync(string languageCode, string? resolvedLanguageCode, bool nativeLanguageSupport) - { - if (!Sldr.IsInitialized) - Sldr.Initialize(); - bool isNative = new LanguageTagService().ConvertToFlores200Code(languageCode, out string internalCode); - Assert.Multiple(() => - { - Assert.That(internalCode, Is.EqualTo(resolvedLanguageCode)); - Assert.That(isNative, Is.EqualTo(nativeLanguageSupport)); - }); - } - - public class TestLanguageTagService : LanguageTagService - { - // Don't call Sldr initialize to call - protected override void InitializeSldrLanguageTags() - { - // remove langtags.json to force download - var cachedAllTagsPath = Path.Combine(Sldr.SldrCachePath, "langtags.json"); - if (File.Exists(cachedAllTagsPath)) - File.Delete(cachedAllTagsPath); - Directory.CreateDirectory(Sldr.SldrCachePath); - } - } - - [Test] - public void BackupLangtagsJsonTest() - { - if (!Sldr.IsInitialized) - Sldr.Initialize(); - var service = new TestLanguageTagService(); - service.ConvertToFlores200Code("en", out string internalCode); - Assert.That(internalCode, Is.EqualTo("eng_Latn")); - } -} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/LocalStorageTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/LocalStorageTests.cs deleted file mode 100644 index 280a54bb1..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/LocalStorageTests.cs +++ /dev/null @@ -1,96 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class LocalStorageTests -{ - [Test] - public async Task ExistsAsync() - { - using var tmpDir = new TempDirectory("test"); - using LocalStorage fs = new(tmpDir.Path); - using (StreamWriter sw = new(await fs.OpenWriteAsync("file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - bool exists = await fs.ExistsAsync("file1"); - Assert.That(exists, Is.True); - } - - [Test] - public async Task OpenReadAsync() - { - using var tmpDir = new TempDirectory("test"); - using LocalStorage fs = new(tmpDir.Path); - string input; - using (StreamWriter sw = new(await fs.OpenWriteAsync("file1"))) - { - input = "Hello"; - sw.WriteLine(input); - } - string? output; - using (StreamReader sr = new(await fs.OpenReadAsync("file1"))) - { - output = sr.ReadLine(); - } - Assert.That(input, Is.EqualTo(output), $"{input} | {output}"); - } - - [Test] - public async Task ListFilesAsync_Recurse() - { - using var tmpDir = new TempDirectory("test"); - using LocalStorage fs = new(tmpDir.Path); - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) - { - string input2 = "Hola"; - sw.WriteLine(input2); - } - IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: true); - Assert.That(files, Is.EquivalentTo(new[] { "test/file1", "test/test/file2" })); - } - - [Test] - public async Task ListFilesAsync_DoNotRecurse() - { - using var tmpDir = new TempDirectory("test"); - using LocalStorage fs = new(tmpDir.Path); - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) - { - string input2 = "Hola"; - sw.WriteLine(input2); - } - IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: false); - Assert.That(files, Is.EquivalentTo(new[] { "test/file1" })); - } - - [Test] - public async Task DeleteFileAsync() - { - using var tmpDir = new TempDirectory("test"); - using LocalStorage fs = new(tmpDir.Path); - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/file1"))) - { - string input = "Hello"; - sw.WriteLine(input); - } - using (StreamWriter sw = new(await fs.OpenWriteAsync("test/test/file2"))) - { - string input2 = "Hola"; - sw.WriteLine(input2); - } - await fs.DeleteAsync("test", recurse: true); - IReadOnlyCollection files = await fs.ListFilesAsync("test", recurse: true); - Assert.That(files, Is.Empty); - } -} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs deleted file mode 100644 index db38ecdb3..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxDeliveryServiceTests.cs +++ /dev/null @@ -1,221 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class MessageOutboxDeliveryServiceTests -{ - private const string OutboxId = "TestOutbox"; - private const string Method1 = "Method1"; - private const string Method2 = "Method2"; - - [Test] - public async Task ProcessMessagesAsync() - { - var env = new TestEnvironment(); - env.AddStandardMessages(); - await env.ProcessMessagesAsync(); - Received.InOrder(() => - { - env.Handler.HandleMessageAsync(Method2, "B", null, Arg.Any()); - env.Handler.HandleMessageAsync(Method1, "A", null, Arg.Any()); - env.Handler.HandleMessageAsync(Method2, "C", null, Arg.Any()); - }); - Assert.That(env.Messages.Count, Is.EqualTo(0)); - } - - [Test] - public async Task ProcessMessagesAsync_Timeout() - { - var env = new TestEnvironment(); - env.AddStandardMessages(); - - // Timeout is long enough where the message attempt will be incremented, but not deleted. - env.EnableHandlerFailure(StatusCode.Internal); - await env.ProcessMessagesAsync(); - // Each group should try to send one message - Assert.That(env.Messages.Get("B").Attempts, Is.EqualTo(1)); - Assert.That(env.Messages.Get("A").Attempts, Is.EqualTo(0)); - Assert.That(env.Messages.Get("C").Attempts, Is.EqualTo(1)); - - // with now shorter timeout, the messages will be deleted. - // 4 start build attempts, and only one build completed attempt - env.Options.CurrentValue.Returns( - new MessageOutboxOptions { MessageExpirationTimeout = TimeSpan.FromMilliseconds(1) } - ); - await env.ProcessMessagesAsync(); - Assert.That(env.Messages.Count, Is.EqualTo(0)); - _ = env - .Handler.Received(1) - .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()); - _ = env - .Handler.Received(4) - .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); - } - - [Test] - public async Task ProcessMessagesAsync_UnavailableFailure() - { - var env = new TestEnvironment(); - env.AddStandardMessages(); - - env.EnableHandlerFailure(StatusCode.Unavailable); - await env.ProcessMessagesAsync(); - // Only the first group should be attempted - but not recorded as attempted - Assert.That(env.Messages.Get("B").Attempts, Is.EqualTo(0)); - Assert.That(env.Messages.Get("A").Attempts, Is.EqualTo(0)); - Assert.That(env.Messages.Get("C").Attempts, Is.EqualTo(0)); - _ = env - .Handler.Received(1) - .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); - - env.Handler.ClearReceivedCalls(); - env.EnableHandlerFailure(StatusCode.Internal); - await env.ProcessMessagesAsync(); - Assert.That(env.Messages.Get("B").Attempts, Is.EqualTo(1)); - Assert.That(env.Messages.Get("A").Attempts, Is.EqualTo(0)); - Assert.That(env.Messages.Get("C").Attempts, Is.EqualTo(1)); - _ = env - .Handler.Received(2) - .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); - - env.Handler.ClearReceivedCalls(); - env.DisableHandlerFailure(); - await env.ProcessMessagesAsync(); - Assert.That(env.Messages.Count, Is.EqualTo(0)); - _ = env - .Handler.Received(1) - .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()); - _ = env - .Handler.Received(2) - .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()); - } - - [Test] - public async Task ProcessMessagesAsync_File() - { - var env = new TestEnvironment(); - env.AddContentStreamMessages(); - - await env.ProcessMessagesAsync(); - Assert.That(env.Messages.Count, Is.EqualTo(0)); - _ = env - .Handler.Received(1) - .HandleMessageAsync(Method1, "A", Arg.Is(s => s != null), Arg.Any()); - env.FileSystem.Received().DeleteFile(Path.Combine("outbox", "A")); - } - - private class TestEnvironment - { - public TestEnvironment() - { - Outboxes = new MemoryRepository(); - Messages = new MemoryRepository(); - - Handler = Substitute.For(); - Handler.OutboxId.Returns(OutboxId); - FileSystem = Substitute.For(); - Options = Substitute.For>(); - Options.CurrentValue.Returns(new MessageOutboxOptions()); - - Service = new MessageOutboxDeliveryService( - Substitute.For(), - [Handler], - FileSystem, - Options, - Substitute.For>() - ); - } - - public MemoryRepository Outboxes { get; } - public MemoryRepository Messages { get; } - public MessageOutboxDeliveryService Service { get; } - public IOutboxMessageHandler Handler { get; } - public IOptionsMonitor Options { get; } - public IFileSystem FileSystem { get; } - - public Task ProcessMessagesAsync() - { - return Service.ProcessMessagesAsync(Messages); - } - - public void AddStandardMessages() - { - // messages out of order - will be fixed when retrieved - Messages.Add( - new OutboxMessage - { - Id = "A", - Index = 2, - Method = Method1, - GroupId = "A", - OutboxRef = OutboxId, - Content = "A", - HasContentStream = false - } - ); - Messages.Add( - new OutboxMessage - { - Id = "B", - Index = 1, - Method = Method2, - OutboxRef = OutboxId, - GroupId = "A", - Content = "B", - HasContentStream = false - } - ); - Messages.Add( - new OutboxMessage - { - Id = "C", - Index = 3, - Method = Method2, - OutboxRef = OutboxId, - GroupId = "B", - Content = "C", - HasContentStream = false - } - ); - } - - public void AddContentStreamMessages() - { - // messages out of order - will be fixed when retrieved - Messages.Add( - new OutboxMessage - { - Id = "A", - Index = 2, - Method = Method1, - GroupId = "A", - OutboxRef = OutboxId, - Content = "A", - HasContentStream = true - } - ); - FileSystem - .OpenRead(Path.Combine("outbox", "A")) - .Returns(ci => new MemoryStream(Encoding.UTF8.GetBytes("Content"))); - } - - public void EnableHandlerFailure(StatusCode code) - { - Handler - .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()) - .ThrowsAsync(new RpcException(new Status(code, ""))); - Handler - .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()) - .ThrowsAsync(new RpcException(new Status(code, ""))); - } - - public void DisableHandlerFailure() - { - Handler - .HandleMessageAsync(Method1, Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(Task.CompletedTask); - Handler - .HandleMessageAsync(Method2, Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(Task.CompletedTask); - } - } -} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxServiceTests.cs deleted file mode 100644 index 7b5d750a1..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/MessageOutboxServiceTests.cs +++ /dev/null @@ -1,93 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class MessageOutboxServiceTests -{ - private const string OutboxId = "TestOutbox"; - private const string Method = "TestMethod"; - - [Test] - public async Task EnqueueMessageAsync_NoContentStream() - { - TestEnvironment env = new(); - - await env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content"); - - Outbox outbox = env.Outboxes.Get(OutboxId); - Assert.That(outbox.CurrentIndex, Is.EqualTo(1)); - - OutboxMessage message = env.Messages.Get("1"); - Assert.That(message.OutboxRef, Is.EqualTo(OutboxId)); - Assert.That(message.Method, Is.EqualTo(Method)); - Assert.That(message.Index, Is.EqualTo(1)); - Assert.That(message.Content, Is.EqualTo("content")); - Assert.That(message.HasContentStream, Is.False); - } - - [Test] - public async Task EnqueueMessageAsync_ExistingOutbox() - { - TestEnvironment env = new(); - env.Outboxes.Add(new Outbox { Id = OutboxId, CurrentIndex = 1 }); - - await env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content"); - - Outbox outbox = env.Outboxes.Get(OutboxId); - Assert.That(outbox.CurrentIndex, Is.EqualTo(2)); - - OutboxMessage message = env.Messages.Get("1"); - Assert.That(message.OutboxRef, Is.EqualTo(OutboxId)); - Assert.That(message.Method, Is.EqualTo(Method)); - Assert.That(message.Index, Is.EqualTo(2)); - Assert.That(message.Content, Is.EqualTo("content")); - Assert.That(message.HasContentStream, Is.False); - } - - [Test] - public async Task EnqueueMessageAsync_HasContentStream() - { - TestEnvironment env = new(); - await using MemoryStream fileStream = new(); - env.FileSystem.OpenWrite(Path.Combine("outbox", "1")).Returns(fileStream); - - await using MemoryStream stream = new(Encoding.UTF8.GetBytes("content")); - await env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content", stream); - - OutboxMessage message = env.Messages.Get("1"); - Assert.That(message.OutboxRef, Is.EqualTo(OutboxId)); - Assert.That(message.Method, Is.EqualTo(Method)); - Assert.That(message.Index, Is.EqualTo(1)); - Assert.That(message.Content, Is.EqualTo("content")); - Assert.That(message.HasContentStream, Is.True); - Assert.That(fileStream.ToArray(), Is.EqualTo(stream.ToArray())); - } - - [Test] - public void EnqueueMessageAsync_ContentTooLarge() - { - TestEnvironment env = new(); - env.Service.MaxDocumentSize = 5; - - Assert.ThrowsAsync(() => env.Service.EnqueueMessageAsync(OutboxId, Method, "A", "content")); - } - - private class TestEnvironment - { - public TestEnvironment() - { - Outboxes = new MemoryRepository(); - Messages = new MemoryRepository(); - var idGenerator = Substitute.For(); - idGenerator.GenerateId().Returns("1"); - FileSystem = Substitute.For(); - var options = Substitute.For>(); - options.CurrentValue.Returns(new MessageOutboxOptions()); - Service = new MessageOutboxService(Outboxes, Messages, idGenerator, FileSystem, options); - } - - public MemoryRepository Outboxes { get; } - public MemoryRepository Messages { get; } - public IFileSystem FileSystem { get; } - public MessageOutboxService Service { get; } - } -} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/ModelCleanupServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/ModelCleanupServiceTests.cs deleted file mode 100644 index 49d50480a..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/ModelCleanupServiceTests.cs +++ /dev/null @@ -1,104 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class ModelCleanupServiceTests -{ - private static readonly List ValidFiles = - [ - "models/engineId1_1.tar.gz", - "models/engineId2_2.tar.gz", - "models/engineId2_3.tar.gz" // only one build ahead - keep - ]; - private static readonly List InvalidFiles = - [ - "models/engineId2_1.tar.gz", // 1 build behind - "models/engineId2_4.tar.gz", // 2 builds ahead - "models/wrongId_1.tar.gz", - "models/engineId1_badBuildNumber.tar.gz", - "models/noBuildNumber.tar.gz", - "models/engineId1_1.differentExtension" - ]; - - [Test] - public async Task CheckModelsAsync_ValidFiles() - { - TestEnvironment env = new(); - await env.CreateFilesAsync(); - - Assert.That( - await env.SharedFileService.ListFilesAsync("models"), - Is.EquivalentTo(ValidFiles.Concat(InvalidFiles)) - ); - await env.CheckModelsAsync(); - // only valid files exist after running service - Assert.That(await env.SharedFileService.ListFilesAsync("models"), Is.EquivalentTo(ValidFiles)); - } - - private class TestEnvironment - { - private readonly MemoryRepository _engines; - - public TestEnvironment() - { - _engines = new MemoryRepository(); - _engines.Add( - new TranslationEngine - { - Id = "engine1", - EngineId = "engineId1", - Type = TranslationEngineType.Nmt, - SourceLanguage = "es", - TargetLanguage = "en", - BuildRevision = 1, - IsModelPersisted = true - } - ); - _engines.Add( - new TranslationEngine - { - Id = "engine2", - EngineId = "engineId2", - Type = TranslationEngineType.Nmt, - SourceLanguage = "es", - TargetLanguage = "en", - BuildRevision = 2, - IsModelPersisted = true - } - ); - - SharedFileService = new SharedFileService(Substitute.For()); - - Service = new ModelCleanupService( - Substitute.For(), - SharedFileService, - Substitute.For>() - ); - } - - public ModelCleanupService Service { get; } - public ISharedFileService SharedFileService { get; } - - public async Task CreateFilesAsync() - { - foreach (string path in ValidFiles) - { - await WriteFileStubAsync(path, "content"); - } - foreach (string path in InvalidFiles) - { - await WriteFileStubAsync(path, "content"); - } - } - - public Task CheckModelsAsync() - { - return Service.CheckModelsAsync(_engines, CancellationToken.None); - } - - private async Task WriteFileStubAsync(string path, string content) - { - using StreamWriter streamWriter = new(await SharedFileService.OpenWriteAsync(path, CancellationToken.None)); - await streamWriter.WriteAsync(content); - } - } -} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/NmtClearMLBuildJobFactoryTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/NmtClearMLBuildJobFactoryTests.cs deleted file mode 100644 index bd69741cb..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/NmtClearMLBuildJobFactoryTests.cs +++ /dev/null @@ -1,123 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class NmtClearMLBuildJobFactoryTests -{ - [Test] - public async Task CreateJobScriptAsync_BuildOptions() - { - var env = new TestEnvironment(); - string script = await env.BuildJobFactory.CreateJobScriptAsync( - "engine1", - "build1", - "test_model", - BuildStage.Train, - buildOptions: "{ \"max_steps\": \"10\" }" - ); - Assert.That( - script, - Is.EqualTo( - @"from machine.jobs.build_nmt_engine import run -args = { - 'model_type': 'test_model', - 'engine_id': 'engine1', - 'build_id': 'build1', - 'src_lang': 'spa_Latn', - 'trg_lang': 'eng_Latn', - 'shared_file_uri': 's3://bucket', - 'shared_file_folder': 'folder1/folder2', - 'build_options': '''{ ""max_steps"": ""10"" }''', - 'clearml': True -} -run(args) -".ReplaceLineEndings("\n") - ) - ); - } - - [Test] - public async Task CreateJobScriptAsync_NoBuildOptions() - { - var env = new TestEnvironment(); - string script = await env.BuildJobFactory.CreateJobScriptAsync( - "engine1", - "build1", - "test_model", - BuildStage.Train - ); - Assert.That( - script, - Is.EqualTo( - @"from machine.jobs.build_nmt_engine import run -args = { - 'model_type': 'test_model', - 'engine_id': 'engine1', - 'build_id': 'build1', - 'src_lang': 'spa_Latn', - 'trg_lang': 'eng_Latn', - 'shared_file_uri': 's3://bucket', - 'shared_file_folder': 'folder1/folder2', - 'clearml': True -} -run(args) -".ReplaceLineEndings("\n") - ) - ); - } - - private class TestEnvironment - { - public ISharedFileService SharedFileService { get; } - public MemoryRepository Engines { get; } - public IOptionsMonitor Options { get; } - public ILanguageTagService LanguageTagService { get; } - public NmtClearMLBuildJobFactory BuildJobFactory { get; } - - public TestEnvironment() - { - Engines = new MemoryRepository(); - Engines.Add( - new TranslationEngine - { - Id = "engine1", - EngineId = "engine1", - Type = TranslationEngineType.Nmt, - SourceLanguage = "es", - TargetLanguage = "en", - BuildRevision = 1, - IsModelPersisted = false, - CurrentBuild = new() - { - BuildId = "build1", - JobId = "job1", - BuildJobRunner = BuildJobRunnerType.ClearML, - Stage = BuildStage.Train, - JobState = BuildJobState.Pending - } - } - ); - Options = Substitute.For>(); - Options.CurrentValue.Returns(new ClearMLOptions { }); - SharedFileService = Substitute.For(); - SharedFileService.GetBaseUri().Returns(new Uri("s3://bucket/folder1/folder2")); - LanguageTagService = Substitute.For(); - LanguageTagService.ConvertToFlores200Code("es", out string spa); - var anyStringArg = Arg.Any(); - LanguageTagService - .ConvertToFlores200Code("es", out anyStringArg) - .Returns(x => - { - x[1] = "spa_Latn"; - return true; - }); - LanguageTagService - .ConvertToFlores200Code("en", out anyStringArg) - .Returns(x => - { - x[1] = "eng_Latn"; - return true; - }); - BuildJobFactory = new NmtClearMLBuildJobFactory(SharedFileService, LanguageTagService, Engines); - } - } -} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/NmtEngineServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/NmtEngineServiceTests.cs deleted file mode 100644 index 1249ad8e9..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/NmtEngineServiceTests.cs +++ /dev/null @@ -1,324 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class NmtEngineServiceTests -{ - [Test] - public async Task StartBuildAsync() - { - using var env = new TestEnvironment(); - TranslationEngine engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildRevision, Is.EqualTo(1)); - await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); - await env.WaitForBuildToFinishAsync(); - engine = env.Engines.Get("engine1"); - Assert.Multiple(() => - { - Assert.That(engine.CurrentBuild, Is.Null); - Assert.That(engine.BuildRevision, Is.EqualTo(2)); - Assert.That(engine.IsModelPersisted, Is.False); - }); - } - - [Test] - public async Task CancelBuildAsync_Building() - { - using var env = new TestEnvironment(); - env.UseInfiniteTrainJob(); - - TranslationEngine engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildRevision, Is.EqualTo(1)); - await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); - await env.WaitForBuildToStartAsync(); - engine = env.Engines.Get("engine1"); - Assert.That(engine.CurrentBuild, Is.Not.Null); - Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); - await env.Service.CancelBuildAsync("engine1"); - await env.WaitForBuildToFinishAsync(); - engine = env.Engines.Get("engine1"); - Assert.That(engine.CurrentBuild, Is.Null); - Assert.That(engine.BuildRevision, Is.EqualTo(1)); - } - - [Test] - public void CancelBuildAsync_NotBuilding() - { - using var env = new TestEnvironment(); - Assert.ThrowsAsync(() => env.Service.CancelBuildAsync("engine1")); - } - - [Test] - public async Task DeleteAsync_WhileBuilding() - { - using var env = new TestEnvironment(); - env.UseInfiniteTrainJob(); - - TranslationEngine engine = env.Engines.Get("engine1"); - Assert.That(engine.BuildRevision, Is.EqualTo(1)); - await env.Service.StartBuildAsync("engine1", "build1", "{}", Array.Empty()); - await env.WaitForBuildToStartAsync(); - engine = env.Engines.Get("engine1"); - Assert.That(engine.CurrentBuild, Is.Not.Null); - Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); - await env.Service.DeleteAsync("engine1"); - // ensure that the train job has completed - await env.WaitForBuildToFinishAsync(); - Assert.That(env.Engines.Contains("engine1"), Is.False); - } - - private class TestEnvironment : ObjectModel.DisposableBase - { - private readonly Hangfire.InMemory.InMemoryStorage _memoryStorage; - private readonly BackgroundJobClient _jobClient; - private BackgroundJobServer _jobServer; - private readonly IDistributedReaderWriterLockFactory _lockFactory; - private readonly CancellationTokenSource _cancellationTokenSource = new(); - private Func _trainJobFunc; - private Task? _trainJobTask; - - public TestEnvironment() - { - if (!Sldr.IsInitialized) - Sldr.Initialize(offlineMode: true); - - _trainJobFunc = RunNormalTrainJob; - Engines = new MemoryRepository(); - Engines.Add( - new TranslationEngine - { - Id = "engine1", - EngineId = "engine1", - Type = TranslationEngineType.Nmt, - SourceLanguage = "es", - TargetLanguage = "en", - BuildRevision = 1, - IsModelPersisted = false - } - ); - _memoryStorage = new Hangfire.InMemory.InMemoryStorage(); - _jobClient = new BackgroundJobClient(_memoryStorage); - PlatformService = Substitute.For(); - _lockFactory = new DistributedReaderWriterLockFactory( - new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), - new MemoryRepository(), - new ObjectIdGenerator() - ); - ClearMLService = Substitute.For(); - ClearMLService - .GetProjectIdAsync("engine1", Arg.Any()) - .Returns(Task.FromResult("project1")); - ClearMLService - .CreateTaskAsync( - "build1", - "project1", - Arg.Any(), - Arg.Any(), - Arg.Any() - ) - .Returns(Task.FromResult("job1")); - ClearMLService - .When(x => x.EnqueueTaskAsync("job1", Arg.Any(), Arg.Any())) - .Do(_ => _trainJobTask = Task.Run(_trainJobFunc)); - ClearMLService - .When(x => x.StopTaskAsync("job1", Arg.Any())) - .Do(_ => _cancellationTokenSource.Cancel()); - SharedFileService = new SharedFileService(Substitute.For()); - var buildJobOptions = Substitute.For>(); - buildJobOptions.CurrentValue.Returns( - new BuildJobOptions - { - ClearML = - [ - new ClearMLBuildQueue() - { - TranslationEngineType = TranslationEngineType.Nmt, - ModelType = "huggingface", - DockerImage = "default", - Queue = "default" - }, - new ClearMLBuildQueue() - { - TranslationEngineType = TranslationEngineType.SmtTransfer, - ModelType = "thot", - DockerImage = "default", - Queue = "default" - } - ] - } - ); - BuildJobService = new BuildJobService( - [ - new HangfireBuildJobRunner(_jobClient, [new NmtHangfireBuildJobFactory()]), - new ClearMLBuildJobRunner( - ClearMLService, - [ - new NmtClearMLBuildJobFactory( - SharedFileService, - Substitute.For(), - Engines - ) - ], - buildJobOptions - ) - ], - Engines - ); - var clearMLOptions = Substitute.For>(); - clearMLOptions.CurrentValue.Returns(new ClearMLOptions()); - ClearMLQueueService = new ClearMLMonitorService( - Substitute.For(), - ClearMLService, - SharedFileService, - clearMLOptions, - buildJobOptions, - Substitute.For>() - ); - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - public NmtEngineService Service { get; private set; } - public IClearMLQueueService ClearMLQueueService { get; } - public MemoryRepository Engines { get; } - public IPlatformService PlatformService { get; } - public IClearMLService ClearMLService { get; } - public ISharedFileService SharedFileService { get; } - public IBuildJobService BuildJobService { get; } - - public void StopServer() - { - _jobServer.Dispose(); - } - - public void StartServer() - { - _jobServer = CreateJobServer(); - Service = CreateService(); - } - - private BackgroundJobServer CreateJobServer() - { - var jobServerOptions = new BackgroundJobServerOptions - { - Activator = new EnvActivator(this), - Queues = new[] { "nmt" }, - CancellationCheckInterval = TimeSpan.FromMilliseconds(50), - }; - return new BackgroundJobServer(jobServerOptions, _memoryStorage); - } - - private NmtEngineService CreateService() - { - return new NmtEngineService( - PlatformService, - _lockFactory, - new MemoryDataAccessContext(), - Engines, - BuildJobService, - new LanguageTagService(), - ClearMLQueueService, - SharedFileService - ); - } - - public async Task WaitForBuildToFinishAsync() - { - await WaitForBuildState(e => e.CurrentBuild is null); - if (_trainJobTask is not null) - await _trainJobTask; - } - - public Task WaitForBuildToStartAsync() - { - return WaitForBuildState(e => - e.CurrentBuild!.JobState is BuildJobState.Active && e.CurrentBuild!.Stage == BuildStage.Train - ); - } - - public void UseInfiniteTrainJob() - { - _trainJobFunc = RunInfiniteTrainJob; - } - - private async Task WaitForBuildState(Func predicate) - { - using ISubscription subscription = await Engines.SubscribeAsync(e => - e.EngineId == "engine1" - ); - while (true) - { - TranslationEngine? engine = subscription.Change.Entity; - if (engine is null || predicate(engine)) - break; - await subscription.WaitForChangeAsync(); - } - } - - private async Task RunNormalTrainJob() - { - await BuildJobService.BuildJobStartedAsync("engine1", "build1"); - - await using Stream stream = await SharedFileService.OpenWriteAsync("builds/build1/pretranslate.trg.json"); - - await BuildJobService.StartBuildJobAsync( - BuildJobRunnerType.Hangfire, - "engine1", - "build1", - BuildStage.Postprocess, - (0, 0.0) - ); - } - - private async Task RunInfiniteTrainJob() - { - await BuildJobService.BuildJobStartedAsync("engine1", "build1"); - - while (!_cancellationTokenSource.IsCancellationRequested) - await Task.Delay(50); - - await BuildJobService.BuildJobFinishedAsync("engine1", "build1", buildComplete: false); - } - - protected override void DisposeManagedResources() - { - _jobServer.Dispose(); - _cancellationTokenSource.Dispose(); - } - - private class EnvActivator(TestEnvironment env) : JobActivator - { - private readonly TestEnvironment _env = env; - - public override object ActivateJob(Type jobType) - { - if (jobType == typeof(NmtPreprocessBuildJob)) - { - return new NmtPreprocessBuildJob( - _env.PlatformService, - _env.Engines, - _env._lockFactory, - new MemoryDataAccessContext(), - Substitute.For>(), - _env.BuildJobService, - _env.SharedFileService, - Substitute.For(), - new LanguageTagService() - ); - } - if (jobType == typeof(PostprocessBuildJob)) - { - return new PostprocessBuildJob( - _env.PlatformService, - _env.Engines, - _env._lockFactory, - new MemoryDataAccessContext(), - _env.BuildJobService, - Substitute.For>(), - _env.SharedFileService - ); - } - return base.ActivateJob(jobType); - } - } - } -} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/PreprocessBuildJobTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/PreprocessBuildJobTests.cs deleted file mode 100644 index 5150cc759..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/PreprocessBuildJobTests.cs +++ /dev/null @@ -1,581 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class PreprocessBuildJobTests -{ - [Test] - public async Task RunAsync_FilterOutEverything() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { }; - - await env.RunBuildJobAsync(corpus1); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(0)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(0)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - - [Test] - public async Task RunAsync_TrainOnAll() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { TrainOnTextIds = null }; - - await env.RunBuildJobAsync(corpus1); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(4)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(1)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - - [Test] - public async Task RunAsync_TrainOnTextIds() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { TrainOnTextIds = ["textId1"] }; - - await env.RunBuildJobAsync(corpus1); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(4)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(1)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - - [Test] - public async Task RunAsync_TrainAndPretranslateAll() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { PretranslateTextIds = null, TrainOnTextIds = null }; - - await env.RunBuildJobAsync(corpus1); - - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(2)); - } - - [Test] - public async Task RunAsync_PretranslateAll() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { PretranslateTextIds = null }; - - await env.RunBuildJobAsync(corpus1); - - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(4)); - } - - [Test] - public async Task RunAsync_PretranslateTextIds() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { PretranslateTextIds = ["textId1"], TrainOnTextIds = null }; - - await env.RunBuildJobAsync(corpus1); - - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(2)); - } - - [Test] - public async Task RunAsync_EnableKeyTerms() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultParatextCorpus with { }; - - await env.RunBuildJobAsync(corpus1, useKeyTerms: true); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(0)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(0)); - Assert.That(termCount, Is.EqualTo(1)); - }); - } - - [Test] - public async Task RunAsync_DisableKeyTerms() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultParatextCorpus with { }; - - await env.RunBuildJobAsync(corpus1, useKeyTerms: false); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(0)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(0)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - - [Test] - public async Task RunAsync_PretranslateChapters() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultParatextCorpus with - { - PretranslateChapters = new Dictionary> - { - { - "1CH", - new HashSet { 12 } - } - } - }; - - await env.RunBuildJobAsync(corpus1); - - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(4)); - } - - [Test] - public async Task RunAsync_TrainOnChapters() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultParatextCorpus with - { - TrainOnChapters = new Dictionary> - { - { - "MAT", - new HashSet { 1 } - } - } - }; - - await env.RunBuildJobAsync(corpus1, useKeyTerms: false); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(5)); - Assert.That(src2Count, Is.EqualTo(0)); - Assert.That(trgCount, Is.EqualTo(0)); - Assert.That(termCount, Is.EqualTo(0)); - }); - } - - [Test] - public async Task RunAsync_MixedSource_Paratext() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultMixedSourceParatextCorpus with - { - TrainOnTextIds = null, - PretranslateTextIds = null - }; - - await env.RunBuildJobAsync(corpus1, useKeyTerms: false); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(4)); - Assert.That(src2Count, Is.EqualTo(12)); - Assert.That(trgCount, Is.EqualTo(1)); - Assert.That(termCount, Is.EqualTo(0)); - }); - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(12)); - } - - [Test] - public async Task RunAsync_MixedSource_Text() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultMixedSourceTextFileCorpus with - { - TrainOnTextIds = null, - PretranslateTextIds = null, - TrainOnChapters = null, - PretranslateChapters = null - }; - - await env.RunBuildJobAsync(corpus1); - - (int src1Count, int src2Count, int trgCount, int termCount) = await env.GetTrainCountAsync(); - Assert.Multiple(() => - { - Assert.That(src1Count, Is.EqualTo(3)); - Assert.That(src2Count, Is.EqualTo(2)); - Assert.That(trgCount, Is.EqualTo(1)); - Assert.That(termCount, Is.EqualTo(0)); - }); - Assert.That(await env.GetPretranslateCountAsync(), Is.EqualTo(2)); - } - - [Test] - public void RunAsync_UnknownLanguageTagsNoData() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { SourceLanguage = "xxx", TargetLanguage = "zzz" }; - - Assert.ThrowsAsync(async () => - { - await env.RunBuildJobAsync(corpus1, engineId: "engine2"); - }); - } - - [Test] - public async Task RunAsync_UnknownLanguageTagsNoDataSmtTransfer() - { - using TestEnvironment env = new(); - Corpus corpus1 = env.DefaultTextFileCorpus with { SourceLanguage = "xxx", TargetLanguage = "zzz" }; - - await env.RunBuildJobAsync(corpus1, engineId: "engine2", engineType: TranslationEngineType.SmtTransfer); - } - - private class TestEnvironment : ObjectModel.DisposableBase - { - private static readonly string TestDataPath = Path.Combine( - AppContext.BaseDirectory, - "..", - "..", - "..", - "Services", - "data" - ); - - private readonly TempDirectory _tempDir; - - public ISharedFileService SharedFileService { get; } - public ICorpusService CorpusService { get; } - public IPlatformService PlatformService { get; } - public MemoryRepository Engines { get; } - public IDistributedReaderWriterLockFactory LockFactory { get; } - public IBuildJobService BuildJobService { get; } - public IClearMLService ClearMLService { get; } - public IOptionsMonitor BuildJobOptions { get; } - - public Corpus DefaultTextFileCorpus { get; } - public Corpus DefaultMixedSourceTextFileCorpus { get; } - public Corpus DefaultParatextCorpus { get; } - public Corpus DefaultMixedSourceParatextCorpus { get; } - - public TestEnvironment() - { - if (!Sldr.IsInitialized) - Sldr.Initialize(offlineMode: true); - - _tempDir = new TempDirectory("PreprocessBuildJobTests"); - - ZipParatextProject("pt-source1"); - ZipParatextProject("pt-source2"); - ZipParatextProject("pt-target1"); - - DefaultTextFileCorpus = new() - { - Id = "corpusId1", - SourceLanguage = "es", - TargetLanguage = "en", - PretranslateTextIds = [], - TrainOnTextIds = [], - SourceFiles = [TextFile("source1")], - TargetFiles = [TextFile("target1")] - }; - - DefaultMixedSourceTextFileCorpus = new() - { - Id = "corpusId1", - SourceLanguage = "es", - TargetLanguage = "en", - PretranslateTextIds = [], - TrainOnTextIds = [], - SourceFiles = [TextFile("source1"), TextFile("source2")], - TargetFiles = [TextFile("target1")] - }; - - DefaultParatextCorpus = new() - { - Id = "corpusId1", - SourceLanguage = "es", - TargetLanguage = "en", - PretranslateTextIds = [], - TrainOnTextIds = [], - SourceFiles = [ParatextFile("pt-source1")], - TargetFiles = [ParatextFile("pt-target1")] - }; - - DefaultMixedSourceParatextCorpus = new() - { - Id = "corpusId1", - SourceLanguage = "es", - TargetLanguage = "en", - PretranslateTextIds = [], - TrainOnTextIds = [], - SourceFiles = [ParatextFile("pt-source1"), ParatextFile("pt-source2")], - TargetFiles = [ParatextFile("pt-target1")] - }; - - Engines = new MemoryRepository(); - Engines.Add( - new TranslationEngine - { - Id = "engine1", - EngineId = "engine1", - Type = TranslationEngineType.Nmt, - SourceLanguage = "es", - TargetLanguage = "en", - BuildRevision = 1, - IsModelPersisted = false, - CurrentBuild = new() - { - BuildId = "build1", - JobId = "job1", - JobState = BuildJobState.Pending, - BuildJobRunner = BuildJobRunnerType.Hangfire, - Stage = BuildStage.Preprocess - } - } - ); - Engines.Add( - new TranslationEngine - { - Id = "engine2", - EngineId = "engine2", - Type = TranslationEngineType.Nmt, - SourceLanguage = "xxx", - TargetLanguage = "zzz", - BuildRevision = 1, - IsModelPersisted = false, - CurrentBuild = new() - { - BuildId = "build1", - JobId = "job1", - JobState = BuildJobState.Pending, - BuildJobRunner = BuildJobRunnerType.Hangfire, - Stage = BuildStage.Preprocess - } - } - ); - Engines.Add( - new TranslationEngine - { - Id = "engine2", - EngineId = "engine2", - Type = TranslationEngineType.Nmt, - SourceLanguage = "xxx", - TargetLanguage = "zzz", - BuildRevision = 1, - IsModelPersisted = false, - CurrentBuild = new() - { - BuildId = "build1", - JobId = "job1", - JobState = BuildJobState.Pending, - BuildJobRunner = BuildJobRunnerType.Hangfire, - Stage = BuildStage.Preprocess - } - } - ); - CorpusService = new CorpusService(); - PlatformService = Substitute.For(); - LockFactory = new DistributedReaderWriterLockFactory( - new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), - new MemoryRepository(), - new ObjectIdGenerator() - ); - BuildJobOptions = Substitute.For>(); - BuildJobOptions.CurrentValue.Returns( - new BuildJobOptions - { - ClearML = - [ - new ClearMLBuildQueue() - { - TranslationEngineType = TranslationEngineType.Nmt, - ModelType = "huggingface", - DockerImage = "default", - Queue = "default" - }, - new ClearMLBuildQueue() - { - TranslationEngineType = TranslationEngineType.SmtTransfer, - ModelType = "thot", - DockerImage = "default", - Queue = "default" - } - ] - } - ); - ClearMLService = Substitute.For(); - ClearMLService - .GetProjectIdAsync("engine1", Arg.Any()) - .Returns(Task.FromResult("project1")); - ClearMLService - .GetProjectIdAsync("engine2", Arg.Any()) - .Returns(Task.FromResult("project1")); - ClearMLService - .GetProjectIdAsync("engine2", Arg.Any()) - .Returns(Task.FromResult("project1")); - ClearMLService - .CreateTaskAsync( - "build1", - "project1", - Arg.Any(), - Arg.Any(), - Arg.Any() - ) - .Returns(Task.FromResult("job1")); - SharedFileService = new SharedFileService(Substitute.For()); - BuildJobService = new BuildJobService( - [ - new HangfireBuildJobRunner( - Substitute.For(), - [new NmtHangfireBuildJobFactory()] - ), - new ClearMLBuildJobRunner( - ClearMLService, - [ - new NmtClearMLBuildJobFactory( - SharedFileService, - Substitute.For(), - Engines - ) - ], - BuildJobOptions - ) - ], - Engines - ); - } - - public PreprocessBuildJob GetBuildJob(TranslationEngineType engineType) - { - switch (engineType) - { - case TranslationEngineType.Nmt: - { - return new NmtPreprocessBuildJob( - PlatformService, - Engines, - LockFactory, - new MemoryDataAccessContext(), - Substitute.For>(), - BuildJobService, - SharedFileService, - CorpusService, - new LanguageTagService() - ) - { - Seed = 1234 - }; - } - case TranslationEngineType.SmtTransfer: - { - return new PreprocessBuildJob( - PlatformService, - Engines, - LockFactory, - new MemoryDataAccessContext(), - Substitute.For>(), - BuildJobService, - SharedFileService, - CorpusService - ) - { - Seed = 1234 - }; - } - default: - throw new InvalidOperationException("Unknown engine type."); - } - ; - } - - public Task RunBuildJobAsync( - Corpus corpus, - bool useKeyTerms = true, - string engineId = "engine1", - TranslationEngineType engineType = TranslationEngineType.Nmt - ) - { - return GetBuildJob(engineType) - .RunAsync(engineId, "build1", [corpus], useKeyTerms ? null : "{\"use_key_terms\":false}", default); - } - - public async Task<(int Source1Count, int Source2Count, int TargetCount, int TermCount)> GetTrainCountAsync() - { - using StreamReader srcReader = new(await SharedFileService.OpenReadAsync("builds/build1/train.src.txt")); - using StreamReader trgReader = new(await SharedFileService.OpenReadAsync("builds/build1/train.trg.txt")); - int src1Count = 0; - int src2Count = 0; - int trgCount = 0; - int termCount = 0; - string? srcLine; - string? trgLine; - while ( - (srcLine = await srcReader.ReadLineAsync()) is not null - && (trgLine = await trgReader.ReadLineAsync()) is not null - ) - { - srcLine = srcLine.Trim(); - trgLine = trgLine.Trim(); - if (srcLine.StartsWith("Source one")) - src1Count++; - else if (srcLine.StartsWith("Source two")) - src2Count++; - else if (srcLine.Length == 0) - trgCount++; - else - termCount++; - } - return (src1Count, src2Count, trgCount, termCount); - } - - public async Task GetPretranslateCountAsync() - { - using StreamReader reader = - new(await SharedFileService.OpenReadAsync("builds/build1/pretranslate.src.json")); - JsonArray? pretranslationJsonObject = JsonSerializer.Deserialize(await reader.ReadToEndAsync()); - return pretranslationJsonObject?.Count ?? 0; - } - - private void ZipParatextProject(string name) - { - ZipFile.CreateFromDirectory(Path.Combine(TestDataPath, name), Path.Combine(_tempDir.Path, $"{name}.zip")); - } - - private CorpusFile ParatextFile(string name) - { - return new() - { - TextId = name, - Format = FileFormat.Paratext, - Location = Path.Combine(_tempDir.Path, $"{name}.zip") - }; - } - - private static CorpusFile TextFile(string name) - { - return new() - { - TextId = "textId1", - Format = FileFormat.Text, - Location = Path.Combine(TestDataPath, $"{name}.txt") - }; - } - - protected override void DisposeManagedResources() - { - _tempDir.Dispose(); - } - } -} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs deleted file mode 100644 index a49a504ae..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/ServalPlatformOutboxMessageHandlerTests.cs +++ /dev/null @@ -1,114 +0,0 @@ -using Google.Protobuf.WellKnownTypes; -using Serval.Translation.V1; - -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class ServalPlatformOutboxMessageHandlerTests -{ - private static readonly JsonSerializerOptions JsonSerializerOptions = - new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; - - [Test] - public async Task HandleMessageAsync_BuildStarted() - { - TestEnvironment env = new(); - - await env.Handler.HandleMessageAsync( - ServalPlatformOutboxConstants.BuildStarted, - JsonSerializer.Serialize(new BuildStartedRequest { BuildId = "C" }), - null - ); - - _ = env.Client.Received(1).BuildStartedAsync(Arg.Is(x => x.BuildId == "C")); - } - - [Test] - public async Task HandleMessageAsync_InsertPretranslations() - { - TestEnvironment env = new(); - await using (MemoryStream stream = new()) - { - await JsonSerializer.SerializeAsync( - stream, - new[] - { - new Pretranslation - { - CorpusId = "corpus1", - TextId = "MAT", - Refs = ["MAT 1:1"], - Translation = "translation" - } - }, - JsonSerializerOptions - ); - stream.Seek(0, SeekOrigin.Begin); - await env.Handler.HandleMessageAsync( - ServalPlatformOutboxConstants.InsertPretranslations, - "engine1", - stream - ); - } - - _ = env.Client.Received(1).InsertPretranslations(); - _ = env - .PretranslationWriter.Received(1) - .WriteAsync( - new InsertPretranslationRequest - { - EngineId = "engine1", - CorpusId = "corpus1", - TextId = "MAT", - Refs = { "MAT 1:1" }, - Translation = "translation" - }, - Arg.Any() - ); - } - - private class TestEnvironment - { - public TestEnvironment() - { - Client = Substitute.For(); - Client.BuildStartedAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); - Client.BuildCanceledAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); - Client.BuildFaultedAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); - Client.BuildCompletedAsync(Arg.Any()).Returns(CreateEmptyUnaryCall()); - Client - .IncrementTranslationEngineCorpusSizeAsync(Arg.Any()) - .Returns(CreateEmptyUnaryCall()); - PretranslationWriter = Substitute.For>(); - Client - .InsertPretranslations(cancellationToken: Arg.Any()) - .Returns( - TestCalls.AsyncClientStreamingCall( - PretranslationWriter, - Task.FromResult(new Empty()), - Task.FromResult(new Metadata()), - () => Status.DefaultSuccess, - () => new Metadata(), - () => { } - ) - ); - - Handler = new ServalPlatformOutboxMessageHandler(Client); - } - - public TranslationPlatformApi.TranslationPlatformApiClient Client { get; } - public ServalPlatformOutboxMessageHandler Handler { get; } - public IClientStreamWriter PretranslationWriter { get; } - - private static AsyncUnaryCall CreateEmptyUnaryCall() - { - return new AsyncUnaryCall( - Task.FromResult(new Empty()), - Task.FromResult(new Metadata()), - () => Status.DefaultSuccess, - () => new Metadata(), - () => { } - ); - } - } -} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/SmtTransferEngineServiceTests.cs b/tests/SIL.Machine.AspNetCore.Tests/Services/SmtTransferEngineServiceTests.cs deleted file mode 100644 index c88a1351e..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/SmtTransferEngineServiceTests.cs +++ /dev/null @@ -1,742 +0,0 @@ -namespace SIL.Machine.AspNetCore.Services; - -[TestFixture] -public class SmtTransferEngineServiceTests -{ - const string EngineId1 = "engine1"; - const string EngineId2 = "engine2"; - const string BuildId1 = "build1"; - const string CorpusId1 = "corpus1"; - - [Test] - public async Task CreateAsync() - { - using var env = new TestEnvironment(); - await env.Service.CreateAsync(EngineId2, "Engine 2", "es", "en"); - TranslationEngine? engine = await env.Engines.GetAsync(e => e.EngineId == EngineId2); - Assert.Multiple(() => - { - Assert.That(engine, Is.Not.Null); - Assert.That(engine?.EngineId, Is.EqualTo(EngineId2)); - Assert.That(engine?.BuildRevision, Is.EqualTo(0)); - Assert.That(engine?.IsModelPersisted, Is.True); - }); - string engineDir = Path.Combine("translation_engines", EngineId2); - _ = env.SmtModelFactory.Received().InitNewAsync(engineDir); - _ = env.TransferEngineFactory.Received().InitNewAsync(engineDir); - } - - [TestCase(BuildJobRunnerType.Hangfire)] - [TestCase(BuildJobRunnerType.ClearML)] - public async Task StartBuildAsync(BuildJobRunnerType trainJobRunnerType) - { - using var env = new TestEnvironment(trainJobRunnerType); - TranslationEngine engine = env.Engines.Get(EngineId1); - Assert.That(engine.BuildRevision, Is.EqualTo(1)); - // ensure that the SMT model was loaded before training - await env.Service.TranslateAsync(EngineId1, n: 1, "esto es una prueba."); - await env.Service.StartBuildAsync( - EngineId1, - BuildId1, - null, - [ - new Corpus() - { - Id = CorpusId1, - SourceLanguage = "es", - TargetLanguage = "en", - SourceFiles = [], - TargetFiles = [], - TrainOnTextIds = null, - PretranslateTextIds = null - } - ] - ); - await env.WaitForBuildToFinishAsync(); - _ = env - .SmtBatchTrainer.Received() - .TrainAsync(Arg.Any>(), Arg.Any()); - _ = env - .TruecaserTrainer.Received() - .TrainAsync(Arg.Any>(), Arg.Any()); - _ = env.SmtBatchTrainer.Received().SaveAsync(Arg.Any()); - _ = env.TruecaserTrainer.Received().SaveAsync(Arg.Any()); - engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Null); - Assert.That(engine.BuildRevision, Is.EqualTo(2)); - // check if SMT model was reloaded upon first use after training - env.SmtModel.ClearReceivedCalls(); - await env.Service.TranslateAsync(EngineId1, n: 1, "esto es una prueba."); - env.SmtModel.Received().Dispose(); - _ = env.SmtModel.DidNotReceive().SaveAsync(); - _ = env.Truecaser.DidNotReceive().SaveAsync(); - } - - [TestCase(BuildJobRunnerType.Hangfire)] - [TestCase(BuildJobRunnerType.ClearML)] - public async Task CancelBuildAsync_Building(BuildJobRunnerType trainJobRunnerType) - { - using var env = new TestEnvironment(trainJobRunnerType); - env.UseInfiniteTrainJob(); - - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); - await env.WaitForTrainingToStartAsync(); - TranslationEngine engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Not.Null); - Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); - await env.Service.CancelBuildAsync(EngineId1); - await env.WaitForBuildToFinishAsync(); - _ = env.SmtBatchTrainer.DidNotReceive().SaveAsync(); - _ = env.TruecaserTrainer.DidNotReceive().SaveAsync(); - engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Null); - } - - [Test] - public void CancelBuildAsync_NotBuilding() - { - using var env = new TestEnvironment(); - Assert.ThrowsAsync(() => env.Service.CancelBuildAsync(EngineId1)); - } - - [Test] - public async Task StartBuildAsync_RestartUnfinishedBuild() - { - using var env = new TestEnvironment(BuildJobRunnerType.Hangfire); - env.UseInfiniteTrainJob(); - - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); - await env.WaitForTrainingToStartAsync(); - TranslationEngine engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Not.Null); - Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); - env.StopServer(); - await env.WaitForBuildToRestartAsync(); - engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Not.Null); - Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Pending)); - _ = env.PlatformService.Received().BuildRestartingAsync(BuildId1); - env.SmtBatchTrainer.ClearSubstitute(ClearOptions.CallActions); - env.StartServer(); - await env.WaitForBuildToFinishAsync(); - engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Null); - } - - [TestCase(BuildJobRunnerType.Hangfire)] - [TestCase(BuildJobRunnerType.ClearML)] - public async Task DeleteAsync_WhileBuilding(BuildJobRunnerType trainJobRunnerType) - { - using var env = new TestEnvironment(trainJobRunnerType); - env.UseInfiniteTrainJob(); - - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); - await env.WaitForTrainingToStartAsync(); - TranslationEngine engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Not.Null); - Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); - await env.Service.DeleteAsync(EngineId1); - await env.WaitForBuildToFinishAsync(); - await env.WaitForAllHangfireJobsToFinishAsync(); - _ = env.SmtBatchTrainer.DidNotReceive().SaveAsync(); - _ = env.TruecaserTrainer.DidNotReceive().SaveAsync(); - Assert.That(env.Engines.Contains(EngineId1), Is.False); - } - - [TestCase(BuildJobRunnerType.Hangfire)] - [TestCase(BuildJobRunnerType.ClearML)] - public async Task TrainSegmentPairAsync(BuildJobRunnerType trainJobRunnerType) - { - using var env = new TestEnvironment(trainJobRunnerType); - env.UseInfiniteTrainJob(); - - await env.Service.StartBuildAsync(EngineId1, BuildId1, "{}", Array.Empty()); - await env.WaitForBuildToStartAsync(); - TranslationEngine engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Not.Null); - Assert.That(engine.CurrentBuild.JobState, Is.EqualTo(BuildJobState.Active)); - await env.Service.TrainSegmentPairAsync(EngineId1, "esto es una prueba.", "this is a test.", true); - env.StopTraining(); - await env.WaitForBuildToFinishAsync(); - engine = env.Engines.Get(EngineId1); - Assert.That(engine.CurrentBuild, Is.Null); - Assert.That(engine.BuildRevision, Is.EqualTo(2)); - _ = env.SmtModel.Received(2).TrainSegmentAsync("esto es una prueba.", "this is a test.", true); - } - - [Test] - public async Task CommitAsync_LoadedInactive() - { - using var env = new TestEnvironment(); - await env.Service.TrainSegmentPairAsync(EngineId1, "esto es una prueba.", "this is a test.", true); - await Task.Delay(10); - await env.CommitAsync(TimeSpan.Zero); - _ = env.SmtModel.Received().SaveAsync(); - Assert.That(env.StateService.Get(EngineId1).IsLoaded, Is.False); - } - - [Test] - public async Task CommitAsync_LoadedActive() - { - using var env = new TestEnvironment(); - await env.Service.TrainSegmentPairAsync(EngineId1, "esto es una prueba.", "this is a test.", true); - await env.CommitAsync(TimeSpan.FromHours(1)); - _ = env.SmtModel.Received().SaveAsync(); - Assert.That(env.StateService.Get(EngineId1).IsLoaded, Is.True); - } - - [Test] - public async Task TranslateAsync() - { - using var env = new TestEnvironment(); - TranslationResult result = (await env.Service.TranslateAsync(EngineId1, n: 1, "esto es una prueba."))[0]; - Assert.That(result.Translation, Is.EqualTo("this is a TEST.")); - } - - [Test] - public async Task GetWordGraphAsync() - { - using var env = new TestEnvironment(); - WordGraph result = await env.Service.GetWordGraphAsync(EngineId1, "esto es una prueba."); - Assert.That( - result.Arcs.Select(a => string.Join(' ', a.TargetTokens)), - Is.EqualTo(new[] { "this is", "a test", "." }) - ); - } - - private class TestEnvironment : ObjectModel.DisposableBase - { - private readonly Hangfire.InMemory.InMemoryStorage _memoryStorage; - private readonly BackgroundJobClient _jobClient; - private BackgroundJobServer _jobServer; - private readonly ITruecaserFactory _truecaserFactory; - private readonly IDistributedReaderWriterLockFactory _lockFactory; - private readonly BuildJobRunnerType _trainJobRunnerType; - private Task? _trainJobTask; - private readonly CancellationTokenSource _cancellationTokenSource = new(); - private bool _training = true; - - public TestEnvironment(BuildJobRunnerType trainJobRunnerType = BuildJobRunnerType.ClearML) - { - _trainJobRunnerType = trainJobRunnerType; - Engines = new MemoryRepository(); - Engines.Add( - new TranslationEngine - { - Id = EngineId1, - EngineId = EngineId1, - Type = TranslationEngineType.SmtTransfer, - SourceLanguage = "es", - TargetLanguage = "en", - BuildRevision = 1, - IsModelPersisted = false - } - ); - TrainSegmentPairs = new MemoryRepository(); - _memoryStorage = new Hangfire.InMemory.InMemoryStorage(); - _jobClient = new BackgroundJobClient(_memoryStorage); - PlatformService = Substitute.For(); - SmtModel = Substitute.For(); - SmtBatchTrainer = Substitute.For(); - SmtBatchTrainer.Stats.Returns( - new TrainStats { TrainCorpusSize = 0, Metrics = { { "bleu", 0.0 }, { "perplexity", 0.0 } } } - ); - Truecaser = Substitute.For(); - TruecaserTrainer = Substitute.For(); - - SmtModelFactory = CreateSmtModelFactory(); - TransferEngineFactory = CreateTransferEngineFactory(); - _truecaserFactory = CreateTruecaserFactory(); - _lockFactory = new DistributedReaderWriterLockFactory( - new OptionsWrapper(new ServiceOptions { ServiceId = "host" }), - new MemoryRepository(), - new ObjectIdGenerator() - ); - SharedFileService = new SharedFileService(Substitute.For()); - var clearMLOptions = Substitute.For>(); - clearMLOptions.CurrentValue.Returns(new ClearMLOptions()); - var buildJobOptions = Substitute.For>(); - buildJobOptions.CurrentValue.Returns( - new BuildJobOptions - { - ClearML = - [ - new ClearMLBuildQueue() - { - TranslationEngineType = TranslationEngineType.Nmt, - ModelType = "huggingface", - DockerImage = "default", - Queue = "default" - }, - new ClearMLBuildQueue() - { - TranslationEngineType = TranslationEngineType.SmtTransfer, - ModelType = "thot", - DockerImage = "default", - Queue = "default" - } - ] - } - ); - ClearMLService = Substitute.For(); - ClearMLService - .GetProjectIdAsync("engine1", Arg.Any()) - .Returns(Task.FromResult("project1")); - ClearMLService - .CreateTaskAsync( - "build1", - "project1", - Arg.Any(), - Arg.Any(), - Arg.Any() - ) - .Returns(Task.FromResult("job1")); - ClearMLService - .When(x => x.EnqueueTaskAsync("job1", Arg.Any(), Arg.Any())) - .Do(_ => _trainJobTask = Task.Run(RunTrainJob)); - ClearMLService - .When(x => x.StopTaskAsync("job1", Arg.Any())) - .Do(_ => _cancellationTokenSource.Cancel()); - ClearMLMonitorService = new ClearMLMonitorService( - Substitute.For(), - ClearMLService, - SharedFileService, - clearMLOptions, - buildJobOptions, - Substitute.For>() - ); - BuildJobService = new BuildJobService( - [ - new HangfireBuildJobRunner(_jobClient, [new SmtTransferHangfireBuildJobFactory()]), - new ClearMLBuildJobRunner( - ClearMLService, - [new SmtTransferClearMLBuildJobFactory(SharedFileService, Engines)], - buildJobOptions - ) - ], - Engines - ); - _jobServer = CreateJobServer(); - StateService = CreateStateService(); - Service = CreateService(); - } - - public SmtTransferEngineService Service { get; private set; } - public SmtTransferEngineStateService StateService { get; private set; } - public MemoryRepository Engines { get; } - public MemoryRepository TrainSegmentPairs { get; } - public ISmtModelFactory SmtModelFactory { get; } - public ITransferEngineFactory TransferEngineFactory { get; } - public ITrainer SmtBatchTrainer { get; } - public IInteractiveTranslationModel SmtModel { get; } - public ITruecaser Truecaser { get; } - public ITrainer TruecaserTrainer { get; } - public IPlatformService PlatformService { get; } - - public IClearMLService ClearMLService { get; } - public IClearMLQueueService ClearMLMonitorService { get; } - - public ISharedFileService SharedFileService { get; } - - public IBuildJobService BuildJobService { get; } - - public async Task CommitAsync(TimeSpan inactiveTimeout) - { - await StateService.CommitAsync(_lockFactory, Engines, inactiveTimeout); - } - - public void StopServer() - { - _jobServer.Dispose(); - StateService.Dispose(); - } - - public void StartServer() - { - _jobServer = CreateJobServer(); - StateService = CreateStateService(); - Service = CreateService(); - } - - public void UseInfiniteTrainJob() - { - SmtBatchTrainer.TrainAsync( - Arg.Any>(), - Arg.Do(cancellationToken => - { - while (_training) - { - cancellationToken.ThrowIfCancellationRequested(); - Thread.Sleep(100); - } - }) - ); - } - - public void StopTraining() - { - _training = false; - } - - private BackgroundJobServer CreateJobServer() - { - var jobServerOptions = new BackgroundJobServerOptions - { - Activator = new EnvActivator(this), - Queues = new[] { "smt_transfer" }, - CancellationCheckInterval = TimeSpan.FromMilliseconds(50), - }; - return new BackgroundJobServer(jobServerOptions, _memoryStorage); - } - - private SmtTransferEngineStateService CreateStateService() - { - var options = Substitute.For>(); - options.CurrentValue.Returns(new SmtTransferEngineOptions()); - return new SmtTransferEngineStateService( - SmtModelFactory, - TransferEngineFactory, - _truecaserFactory, - options - ); - } - - private SmtTransferEngineService CreateService() - { - return new SmtTransferEngineService( - _lockFactory, - PlatformService, - new MemoryDataAccessContext(), - Engines, - TrainSegmentPairs, - StateService, - BuildJobService, - ClearMLMonitorService - ); - } - - private ISmtModelFactory CreateSmtModelFactory() - { - ISmtModelFactory factory = Substitute.For(); - - var translationResult = new TranslationResult( - "this is a TEST.", - "esto es una prueba .".Split(), - "this is a TEST .".Split(), - [1.0, 1.0, 1.0, 1.0, 1.0], - [ - TranslationSources.Smt, - TranslationSources.Smt, - TranslationSources.Smt, - TranslationSources.Smt, - TranslationSources.Smt - ], - new WordAlignmentMatrix(5, 5) - { - [0, 0] = true, - [1, 1] = true, - [2, 2] = true, - [3, 3] = true, - [4, 4] = true - }, - [new Phrase(Range.Create(0, 5), 5)] - ); - SmtModel - .TranslateAsync(1, Arg.Any()) - .Returns(Task.FromResult>([translationResult])); - SmtModel - .GetWordGraphAsync(Arg.Any()) - .Returns( - Task.FromResult( - new WordGraph( - "esto es una prueba .".Split(), - new[] - { - new WordGraphArc( - 0, - 1, - 1.0, - "this is".Split(), - new WordAlignmentMatrix(2, 2) { [0, 0] = true, [1, 1] = true }, - Range.Create(0, 2), - GetSources(2, false), - [1.0, 1.0] - ), - new WordGraphArc( - 1, - 2, - 1.0, - "a test".Split(), - new WordAlignmentMatrix(2, 2) { [0, 0] = true, [1, 1] = true }, - Range.Create(2, 4), - GetSources(2, false), - [1.0, 1.0] - ), - new WordGraphArc( - 2, - 3, - 1.0, - ".".Split(), - new WordAlignmentMatrix(1, 1) { [0, 0] = true }, - Range.Create(4, 5), - GetSources(1, false), - [1.0] - ) - }, - [3] - ) - ) - ); - - factory - .CreateAsync( - Arg.Any(), - Arg.Any>(), - Arg.Any>(), - Arg.Any(), - Arg.Any() - ) - .Returns(Task.FromResult(SmtModel)); - factory - .CreateTrainerAsync( - Arg.Any(), - Arg.Any>(), - Arg.Any(), - Arg.Any() - ) - .Returns(Task.FromResult(SmtBatchTrainer)); - return factory; - } - - private static ITransferEngineFactory CreateTransferEngineFactory() - { - ITransferEngineFactory factory = Substitute.For(); - ITranslationEngine engine = Substitute.For(); - engine - .TranslateAsync(Arg.Any()) - .Returns( - Task.FromResult( - new TranslationResult( - "this is a TEST.", - "esto es una prueba .".Split(), - "this is a TEST .".Split(), - [1.0, 1.0, 1.0, 1.0, 1.0], - [ - TranslationSources.Transfer, - TranslationSources.Transfer, - TranslationSources.Transfer, - TranslationSources.Transfer, - TranslationSources.Transfer - ], - new WordAlignmentMatrix(5, 5) - { - [0, 0] = true, - [1, 1] = true, - [2, 2] = true, - [3, 3] = true, - [4, 4] = true - }, - [new Phrase(Range.Create(0, 5), 5)] - ) - ) - ); - factory - .CreateAsync( - Arg.Any(), - Arg.Any>(), - Arg.Any>(), - Arg.Any(), - Arg.Any() - ) - .Returns(Task.FromResult(engine)); - return factory; - } - - private ITruecaserFactory CreateTruecaserFactory() - { - ITruecaserFactory factory = Substitute.For(); - factory.CreateAsync(Arg.Any()).Returns(Task.FromResult(Truecaser)); - factory - .CreateTrainerAsync( - Arg.Any(), - Arg.Any>(), - Arg.Any(), - Arg.Any() - ) - .Returns(Task.FromResult(TruecaserTrainer)); - return factory; - } - - private static TranslationSources[] GetSources(int count, bool isUnknown) - { - var sources = new TranslationSources[count]; - for (int i = 0; i < count; i++) - sources[i] = isUnknown ? TranslationSources.None : TranslationSources.Smt; - return sources; - } - - public async Task WaitForAllHangfireJobsToFinishAsync() - { - IMonitoringApi monitoringApi = _memoryStorage.GetMonitoringApi(); - while (monitoringApi.EnqueuedCount("smt_transfer") > 0 || monitoringApi.ProcessingCount() > 0) - await Task.Delay(50); - } - - public async Task WaitForBuildToFinishAsync() - { - await WaitForBuildState(e => e.CurrentBuild is null); - if (_trainJobTask is not null) - await _trainJobTask; - } - - public Task WaitForBuildToStartAsync() - { - return WaitForBuildState(e => e.CurrentBuild!.JobState is BuildJobState.Active); - } - - public Task WaitForTrainingToStartAsync() - { - return WaitForBuildState(e => - e.CurrentBuild!.JobState is BuildJobState.Active && e.CurrentBuild!.Stage is BuildStage.Train - ); - } - - public Task WaitForBuildToRestartAsync() - { - return WaitForBuildState(e => e.CurrentBuild!.JobState is BuildJobState.Pending); - } - - private async Task WaitForBuildState(Func predicate) - { - using ISubscription subscription = await Engines.SubscribeAsync(e => - e.EngineId == EngineId1 - ); - while (true) - { - TranslationEngine? engine = subscription.Change.Entity; - if (engine is null || predicate(engine)) - break; - await subscription.WaitForChangeAsync(); - } - } - - protected override void DisposeManagedResources() - { - StateService.Dispose(); - _jobServer.Dispose(); - } - - private async Task RunTrainJob() - { - try - { - await BuildJobService.BuildJobStartedAsync("engine1", "build1", _cancellationTokenSource.Token); - - string engineDir = Path.Combine("translation_engines", EngineId1); - await SmtModelFactory.InitNewAsync(engineDir, _cancellationTokenSource.Token); - ITextCorpus sourceCorpus = new DictionaryTextCorpus(); - ITextCorpus targetCorpus = new DictionaryTextCorpus(); - IParallelTextCorpus parallelCorpus = sourceCorpus.AlignRows(targetCorpus); - LatinWordTokenizer tokenizer = new(); - using ITrainer smtModelTrainer = await SmtModelFactory.CreateTrainerAsync( - engineDir, - tokenizer, - parallelCorpus, - _cancellationTokenSource.Token - ); - using ITrainer truecaseTrainer = await _truecaserFactory.CreateTrainerAsync( - engineDir, - tokenizer, - targetCorpus, - _cancellationTokenSource.Token - ); - await smtModelTrainer.TrainAsync(null, _cancellationTokenSource.Token); - await truecaseTrainer.TrainAsync(cancellationToken: _cancellationTokenSource.Token); - - await smtModelTrainer.SaveAsync(_cancellationTokenSource.Token); - await truecaseTrainer.SaveAsync(_cancellationTokenSource.Token); - - await using Stream engineStream = await SharedFileService.OpenWriteAsync( - $"builds/{BuildId1}/model.tar.gz", - _cancellationTokenSource.Token - ); - - await using Stream targetStream = await SharedFileService.OpenWriteAsync( - $"builds/{BuildId1}/pretranslate.trg.json", - _cancellationTokenSource.Token - ); - - await BuildJobService.StartBuildJobAsync( - BuildJobRunnerType.Hangfire, - EngineId1, - BuildId1, - BuildStage.Postprocess, - data: (0, 0.0) - ); - } - catch (OperationCanceledException) - { - await BuildJobService.BuildJobFinishedAsync("engine1", "build1", buildComplete: false); - } - } - - private class EnvActivator(TestEnvironment env) : JobActivator - { - private readonly TestEnvironment _env = env; - - public override object ActivateJob(Type jobType) - { - if (jobType == typeof(PreprocessBuildJob)) - { - return new PreprocessBuildJob( - _env.PlatformService, - _env.Engines, - _env._lockFactory, - new MemoryDataAccessContext(), - Substitute.For>(), - _env.BuildJobService, - _env.SharedFileService, - Substitute.For() - ) - { - TrainJobRunnerType = _env._trainJobRunnerType - }; - } - if (jobType == typeof(SmtTransferPostprocessBuildJob)) - { - var options = Substitute.For>(); - options.CurrentValue.Returns(new SmtTransferEngineOptions()); - return new SmtTransferPostprocessBuildJob( - _env.PlatformService, - _env.Engines, - _env._lockFactory, - new MemoryDataAccessContext(), - _env.BuildJobService, - Substitute.For>(), - _env.SharedFileService, - _env.TrainSegmentPairs, - _env.SmtModelFactory, - _env._truecaserFactory, - options - ); - } - if (jobType == typeof(SmtTransferTrainBuildJob)) - { - return new SmtTransferTrainBuildJob( - _env.PlatformService, - _env.Engines, - _env._lockFactory, - new MemoryDataAccessContext(), - _env.BuildJobService, - Substitute.For>(), - _env.SharedFileService, - _env._truecaserFactory, - _env.SmtModelFactory, - _env.TransferEngineFactory - ); - } - return base.ActivateJob(jobType); - } - } - } -} diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/04LEVTe1.SFM b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/04LEVTe1.SFM deleted file mode 100644 index b86652904..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/04LEVTe1.SFM +++ /dev/null @@ -1,8 +0,0 @@ -\id LEV - Test -\h Leviticus -\mt Leviticus -\c 14 -\p -\v 55 Source one, chapter fourteen, verse fifty-five. -\v 55b Segment b. -\v 56 Source one, chapter fourteen, verse fifty-six. diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/131CHTe1.SFM b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/131CHTe1.SFM deleted file mode 100644 index 4eb8b5fd2..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/131CHTe1.SFM +++ /dev/null @@ -1,12 +0,0 @@ -\id 1CH - Test -\h 1 Chronicles -\mt 1 Chronicles -\c 12 -\p -\v 1 Source one, chapter twelve, verse one. -\v 2 Source one, chapter twelve, verse two. -\v 3-7 Source one, chapter twelve, verses three through seven. -\v 8 Source one, chapter twelve, verse eight. -\c 13 -\p -\v 1 Source one, chapter thirteen, verse one. diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/41MATTe1.SFM b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/41MATTe1.SFM deleted file mode 100644 index ccf166e28..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/41MATTe1.SFM +++ /dev/null @@ -1,16 +0,0 @@ -\id MAT - Test -\h Matthew -\mt Matthew -\ip An introduction to Matthew -\c 1 -\p -\v 1 Source one, chapter one, verse one. -\v 2-3 Source one, chapter one, verse two and three. -\v 4 Source one, chapter one, verse four. -\v 5 Source one, chapter one, verse five. -\v 6 Source one, chapter one, verse six. -\v 7-9 Source one, chapter one, verse seven, eight, and nine. -\v 10 Source one, chapter one, verse ten. -\c 2 -\p -\v 1 Source one, chapter two, verse one. diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/42MRKTe1.SFM b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/42MRKTe1.SFM deleted file mode 100644 index ff8aaf6e2..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/42MRKTe1.SFM +++ /dev/null @@ -1,4 +0,0 @@ -\id MRK - Test -\h Mark -\mt Mark -\ip An introduction to Mark diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/Settings.xml b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/Settings.xml deleted file mode 100644 index c80caedfa..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/Settings.xml +++ /dev/null @@ -1,34 +0,0 @@ - - usfm.sty - 4 - en::: - English - 8.0.100.76 - Test1 - 65001 - T - - NFC - Te1 - a7e0b3ce0200736062f9f810a444dbfbe64aca35 - Charis SIL - 12 - - - - 41MAT - - Tes.SFM - Major::BiblicalTerms.xml - F - F - F - Public - Standard:: - - 3 - 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000 - - - \ No newline at end of file diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/TermRenderings.xml b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/TermRenderings.xml deleted file mode 100644 index 03e450204..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/TermRenderings.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - Abraham - - - - - - diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/custom.vrs b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/custom.vrs deleted file mode 100644 index 9c1cd3873..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source1/custom.vrs +++ /dev/null @@ -1,31 +0,0 @@ -# custom.vrs - -LEV 14:56 -ROM 14:26 -REV 12:17 -TOB 5:22 -TOB 10:12 -SIR 23:28 -ESG 1:22 -ESG 3:15 -ESG 5:14 -ESG 8:17 -ESG 10:14 -SIR 33:33 -SIR 41:24 -BAR 1:22 -4MA 7:25 -4MA 12:20 - -# deliberately missing verses --ROM 16:26 --ROM 16:27 --3JN 1:15 --S3Y 1:49 --ESG 4:6 --ESG 9:5 --ESG 9:30 - -LEV 14:55 = LEV 14:55 -LEV 14:55 = LEV 14:56 -LEV 14:56 = LEV 14:57 diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/04LEVTe3.SFM b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/04LEVTe3.SFM deleted file mode 100644 index a0edc38b0..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/04LEVTe3.SFM +++ /dev/null @@ -1,7 +0,0 @@ -\id LEV - Test -\h Leviticus -\mt Leviticus -\c 14 -\p -\v 55 Source two, chapter fourteen, verse fifty-five. -\v 56 Source two, chapter fourteen, verse fifty-six. diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/131CHTe3.SFM b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/131CHTe3.SFM deleted file mode 100644 index 05cdff2ca..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/131CHTe3.SFM +++ /dev/null @@ -1,16 +0,0 @@ -\id 1CH - Test -\h 1 Chronicles -\mt 1 Chronicles -\c 12 -\p -\v 1 Source two, chapter twelve, verse one. -\v 2 Source two, chapter twelve, verse two. -\v 3 Source two, chapter twelve, verse three. -\v 4 Source two, chapter twelve, verse four. -\v 5 Source two, chapter twelve, verse five. -\v 6 Source two, chapter twelve, verse six. -\v 7 Source two, chapter twelve, verse seven. -\v 8 Source two, chapter twelve, verse eight. -\c 13 -\p -\v 1 Source two, chapter thirteen, verse one. diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/41MATTe3.SFM b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/41MATTe3.SFM deleted file mode 100644 index 7208a72dd..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/41MATTe3.SFM +++ /dev/null @@ -1,19 +0,0 @@ -\id MAT - Test -\h Matthew -\mt Matthew -\ip An introduction to Matthew -\c 1 -\p -\v 1 Source two, chapter one, verse one. -\v 2 Source two, chapter one, verse two. -\v 3 Source two, chapter one, verse three. -\v 4 Source two, chapter one, verse four. -\v 5 Source two, chapter one, verse five. -\v 6 Source two, chapter one, verse six. -\v 7 Source two, chapter one, verse seven. -\v 8 Source two, chapter one, verse eight. -\v 9 Source two, chapter one, verse nine. -\v 10 Source two, chapter one, verse ten. -\c 2 -\p -\v 1 Source two, chapter two, verse one. diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/42MRKTe3.SFM b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/42MRKTe3.SFM deleted file mode 100644 index 223809831..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/42MRKTe3.SFM +++ /dev/null @@ -1,7 +0,0 @@ -\id MRK - Test -\h Mark -\mt Mark -\ip An introduction to Mark -\c 1 -\p -\v 1 Source two, chapter one, verse one. diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/Settings.xml b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/Settings.xml deleted file mode 100644 index affce0ec4..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/Settings.xml +++ /dev/null @@ -1,34 +0,0 @@ - - usfm.sty - 4 - en::: - English - 8.0.100.76 - Test3 - 65001 - T - - NFC - Te3 - a7e0b3ce0200736062f9f810a444dbfbe64aca35 - Charis SIL - 12 - - - - 41MAT - - Tes.SFM - Major::BiblicalTerms.xml - F - F - F - Public - Standard:: - - 3 - 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000 - - - \ No newline at end of file diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/TermRenderings.xml b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/TermRenderings.xml deleted file mode 100644 index 03e450204..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/TermRenderings.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - Abraham - - - - - - diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/custom.vrs b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/custom.vrs deleted file mode 100644 index 9c1cd3873..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-source2/custom.vrs +++ /dev/null @@ -1,31 +0,0 @@ -# custom.vrs - -LEV 14:56 -ROM 14:26 -REV 12:17 -TOB 5:22 -TOB 10:12 -SIR 23:28 -ESG 1:22 -ESG 3:15 -ESG 5:14 -ESG 8:17 -ESG 10:14 -SIR 33:33 -SIR 41:24 -BAR 1:22 -4MA 7:25 -4MA 12:20 - -# deliberately missing verses --ROM 16:26 --ROM 16:27 --3JN 1:15 --S3Y 1:49 --ESG 4:6 --ESG 9:5 --ESG 9:30 - -LEV 14:55 = LEV 14:55 -LEV 14:55 = LEV 14:56 -LEV 14:56 = LEV 14:57 diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/41MATTe2.SFM b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/41MATTe2.SFM deleted file mode 100644 index 69f462508..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/41MATTe2.SFM +++ /dev/null @@ -1,17 +0,0 @@ -\id MAT - Test -\h Matthew -\mt Matthew -\ip An introduction to Matthew -\c 1 -\p -\v 1 Target one, chapter one, verse one. -\v 2 Target one, chapter one, verse two. -\v 3 Target one, chapter one, verse three. -\v 4 -\v 5-6 Target one, chapter one, verse five and six. -\v 7-8 Target one, chapter one, verse seven and eight. -\v 9-10 Target one, chapter one, verse nine and ten. -\c 2 -\p -\v 1 Target one, chapter two, verse one. -\v 2 Target one, chapter two, verse two. diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/42MRKTe2.SFM b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/42MRKTe2.SFM deleted file mode 100644 index 460009633..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/42MRKTe2.SFM +++ /dev/null @@ -1,4 +0,0 @@ -\id MRK - Test -\h Mark -\mt Mark -\ip An introduction to Mark \ No newline at end of file diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/Settings.xml b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/Settings.xml deleted file mode 100644 index 37d2772a0..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/Settings.xml +++ /dev/null @@ -1,33 +0,0 @@ - - usfm.sty - 4 - en::: - English - 8.0.100.76 - Test2 - 65001 - T - - NFC - Te2 - a7e0b3ce0200736062f9f810a444dbfbe64aca35 - Charis SIL - 12 - - - - 41MAT - - Ten.SFM - F - F - F - Public - Standard:: - - 3 - 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 - 000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000 - - - \ No newline at end of file diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/TermRenderings.xml b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/TermRenderings.xml deleted file mode 100644 index 03e450204..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/TermRenderings.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - Abraham - - - - - - diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/custom.vrs b/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/custom.vrs deleted file mode 100644 index 9c1cd3873..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/pt-target1/custom.vrs +++ /dev/null @@ -1,31 +0,0 @@ -# custom.vrs - -LEV 14:56 -ROM 14:26 -REV 12:17 -TOB 5:22 -TOB 10:12 -SIR 23:28 -ESG 1:22 -ESG 3:15 -ESG 5:14 -ESG 8:17 -ESG 10:14 -SIR 33:33 -SIR 41:24 -BAR 1:22 -4MA 7:25 -4MA 12:20 - -# deliberately missing verses --ROM 16:26 --ROM 16:27 --3JN 1:15 --S3Y 1:49 --ESG 4:6 --ESG 9:5 --ESG 9:30 - -LEV 14:55 = LEV 14:55 -LEV 14:55 = LEV 14:56 -LEV 14:56 = LEV 14:57 diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/source1.txt b/tests/SIL.Machine.AspNetCore.Tests/Services/data/source1.txt deleted file mode 100644 index 2aeb971ce..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/source1.txt +++ /dev/null @@ -1,7 +0,0 @@ -Source one, Line 1 -Source one, Line 2 - -Source one, Line 4 - -Source one, Line 6 - diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/source2.txt b/tests/SIL.Machine.AspNetCore.Tests/Services/data/source2.txt deleted file mode 100644 index 7f4a06693..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/source2.txt +++ /dev/null @@ -1,7 +0,0 @@ -Source two, Line 1 -Source two, Line 2 - -Source two, Line 4 -Source two, Line 5 -Source two, Line 6 - diff --git a/tests/SIL.Machine.AspNetCore.Tests/Services/data/target1.txt b/tests/SIL.Machine.AspNetCore.Tests/Services/data/target1.txt deleted file mode 100644 index 816e94355..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Services/data/target1.txt +++ /dev/null @@ -1,7 +0,0 @@ -Target one, Line 1 - - -Target one, Line 4 - - -Target one, Line 7 diff --git a/tests/SIL.Machine.AspNetCore.Tests/Usings.cs b/tests/SIL.Machine.AspNetCore.Tests/Usings.cs deleted file mode 100644 index 94f9c3a80..000000000 --- a/tests/SIL.Machine.AspNetCore.Tests/Usings.cs +++ /dev/null @@ -1,27 +0,0 @@ -global using System.IO.Compression; -global using System.Text; -global using System.Text.Json; -global using System.Text.Json.Nodes; -global using Grpc.Core; -global using Grpc.Core.Testing; -global using Hangfire; -global using Hangfire.Storage; -global using Microsoft.Extensions.Hosting; -global using Microsoft.Extensions.Hosting.Internal; -global using Microsoft.Extensions.Logging; -global using Microsoft.Extensions.Options; -global using NSubstitute; -global using NSubstitute.ClearExtensions; -global using NSubstitute.ExceptionExtensions; -global using NSubstitute.ReceivedExtensions; -global using NUnit.Framework; -global using RichardSzalay.MockHttp; -global using SIL.DataAccess; -global using SIL.Machine.Annotations; -global using SIL.Machine.AspNetCore.Configuration; -global using SIL.Machine.AspNetCore.Models; -global using SIL.Machine.Corpora; -global using SIL.Machine.Tokenization; -global using SIL.Machine.Translation; -global using SIL.Machine.Utils; -global using SIL.WritingSystems;