Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
ad946b2
commit 8e0d367
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| namespace Serval.Machine.Shared.Configuration; | ||
|
|
||
| public class WordAlignmentModelOptions | ||
| { | ||
| public const string Key = "WordAlignmentModel"; | ||
|
|
||
| public WordAlignmentModelOptions() | ||
| { | ||
| string installDir = Path.GetDirectoryName(Assembly.GetEntryAssembly()!.Location)!; | ||
| NewModelFile = Path.Combine(installDir, "thot-new-model.zip"); | ||
| } | ||
|
|
||
| public string NewModelFile { get; set; } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| namespace Serval.Machine.Shared.Services; | ||
|
|
||
| public interface IModelFactory | ||
| { | ||
| ITrainer CreateTrainer( | ||
| string engineDir, | ||
| IRangeTokenizer<string, int, string> tokenizer, | ||
| IParallelTextCorpus corpus | ||
| ); | ||
|
|
||
| void InitNew(string engineDir); | ||
| void Cleanup(string engineDir); | ||
| Task UpdateEngineFromAsync(string engineDir, Stream source, CancellationToken cancellationToken = default); | ||
| Task SaveEngineToAsync(string engineDir, Stream destination, CancellationToken cancellationToken = default); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| namespace Serval.Machine.Shared.Services; | ||
|
|
||
| public interface IWordAlignmentModelFactory | ||
| { | ||
| IWordAlignmentModel Create(string engineDir, string modelType); | ||
| ITrainer CreateTrainer( | ||
| string engineDir, | ||
| string modelType, | ||
| ITokenizer<string, int, string> tokenizer, | ||
| IParallelTextCorpus corpus | ||
| ); | ||
| void InitNew(string engineDir); | ||
| void Cleanup(string engineDir); | ||
| Task UpdateEngineFromAsync(string engineDir, Stream source, CancellationToken cancellationToken = default); | ||
| Task SaveEngineToAsync(string engineDir, Stream destination, CancellationToken cancellationToken = default); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| namespace Serval.Machine.Shared.Services; | ||
|
|
||
| public abstract class ModelFactoryBase : IModelFactory | ||
| { | ||
| public virtual ITrainer CreateTrainer( | ||
| string engineDir, | ||
| IRangeTokenizer<string, int, string> tokenizer, | ||
| IParallelTextCorpus corpus | ||
| ) | ||
| { | ||
| throw new NotImplementedException(); | ||
| } | ||
|
|
||
| public virtual void InitNew(string engineDir) | ||
| { | ||
| throw new NotImplementedException(); | ||
| } | ||
|
|
||
| public void Cleanup(string engineDir) | ||
| { | ||
| if (!Directory.Exists(engineDir)) | ||
| return; | ||
| DirectoryHelper.DeleteDirectoryRobust(Path.Combine(engineDir, "lm")); | ||
| DirectoryHelper.DeleteDirectoryRobust(Path.Combine(engineDir, "tm")); | ||
| string smtConfigFileName = Path.Combine(engineDir, "smt.cfg"); | ||
| if (File.Exists(smtConfigFileName)) | ||
| File.Delete(smtConfigFileName); | ||
| if (!Directory.EnumerateFileSystemEntries(engineDir).Any()) | ||
| Directory.Delete(engineDir); | ||
| } | ||
|
|
||
| public async Task UpdateEngineFromAsync( | ||
| string engineDir, | ||
| Stream source, | ||
| CancellationToken cancellationToken = default | ||
| ) | ||
| { | ||
| if (!Directory.Exists(engineDir)) | ||
| Directory.CreateDirectory(engineDir); | ||
|
|
||
| await using MemoryStream memoryStream = new(); | ||
| await using (GZipStream gzipStream = new(source, CompressionMode.Decompress)) | ||
| { | ||
| await gzipStream.CopyToAsync(memoryStream, cancellationToken); | ||
| } | ||
| memoryStream.Seek(0, SeekOrigin.Begin); | ||
| await TarFile.ExtractToDirectoryAsync( | ||
| memoryStream, | ||
| engineDir, | ||
| overwriteFiles: true, | ||
| cancellationToken: cancellationToken | ||
| ); | ||
| } | ||
|
|
||
| public async Task SaveEngineToAsync( | ||
| string engineDir, | ||
| Stream destination, | ||
| CancellationToken cancellationToken = default | ||
| ) | ||
| { | ||
| // create zip archive in memory stream | ||
| // This cannot be created directly to the shared stream because it all needs to be written at once | ||
| await using MemoryStream memoryStream = new(); | ||
| await TarFile.CreateFromDirectoryAsync( | ||
| engineDir, | ||
| memoryStream, | ||
| includeBaseDirectory: false, | ||
| cancellationToken: cancellationToken | ||
| ); | ||
| memoryStream.Seek(0, SeekOrigin.Begin); | ||
| await using GZipStream gzipStream = new(destination, CompressionMode.Compress); | ||
| await memoryStream.CopyToAsync(gzipStream, cancellationToken); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.