422 on badly formatted text file; updated docs on create-file --ECL

src/Serval.Client/Client.g.cs

@@ -47,8 +47,11 @@ public partial interface IDataFilesClient
         /// <param name="file">The file to upload.  Max size: 100MB</param>
         /// <param name="format">File format options:
         /// <br/>* **Text**: One translation unit (a.k.a., verse) per line
-        /// <br/>  * If there is a tab, the content before the tab is the unique identifier for the line
-        /// <br/>  * Otherwise, no tabs should be used in the file.
+        /// <br/>  * If a line contains a tab, characters before the tab are used as a unique identifier for the line, characters after the tab are understood as the content of the verse, and if there is another tab following the verse content, characters after this second tab are assumed to be column codes like "ss" etc. for sectioning and other formatting. See this example of a tab-delimited text file:
+        /// <br/>    &gt; verse_001_005	Ὑπομνῆσαι δὲ ὑμᾶς βούλομαι , εἰδότας ὑμᾶς ἅπαξ τοῦτο
+        /// <br/>    &gt; verse_001_006	Ἀγγέλους τε τοὺς μὴ τηρήσαντας τὴν ἑαυτῶν ἀρχήν , ἀλλὰ	ss
+        /// <br/>    &gt; verse_001_007	Ὡς Σόδομα καὶ Γόμορρα , καὶ αἱ περὶ αὐτὰς πόλεις    ss
+        /// <br/>  * Otherwise, *no tabs* should be used in the file and a unique identifier will generated for each translation unit based on the linenumber.
         /// <br/>* **Paratext**: A complete, zipped Paratext project</param>
         /// <param name="name">A name to help identify and distinguish the file.
         /// <br/>Recommendation: Create a multi-part name to distinguish between projects, uses, languages, etc.
@@ -231,8 +234,11 @@ public string BaseUrl
         /// <param name="file">The file to upload.  Max size: 100MB</param>
         /// <param name="format">File format options:
         /// <br/>* **Text**: One translation unit (a.k.a., verse) per line
-        /// <br/>  * If there is a tab, the content before the tab is the unique identifier for the line
-        /// <br/>  * Otherwise, no tabs should be used in the file.
+        /// <br/>  * If a line contains a tab, characters before the tab are used as a unique identifier for the line, characters after the tab are understood as the content of the verse, and if there is another tab following the verse content, characters after this second tab are assumed to be column codes like "ss" etc. for sectioning and other formatting. See this example of a tab-delimited text file:
+        /// <br/>    &gt; verse_001_005	Ὑπομνῆσαι δὲ ὑμᾶς βούλομαι , εἰδότας ὑμᾶς ἅπαξ τοῦτο
+        /// <br/>    &gt; verse_001_006	Ἀγγέλους τε τοὺς μὴ τηρήσαντας τὴν ἑαυτῶν ἀρχήν , ἀλλὰ	ss
+        /// <br/>    &gt; verse_001_007	Ὡς Σόδομα καὶ Γόμορρα , καὶ αἱ περὶ αὐτὰς πόλεις    ss
+        /// <br/>  * Otherwise, *no tabs* should be used in the file and a unique identifier will generated for each translation unit based on the linenumber.
         /// <br/>* **Paratext**: A complete, zipped Paratext project</param>
         /// <param name="name">A name to help identify and distinguish the file.
         /// <br/>Recommendation: Create a multi-part name to distinguish between projects, uses, languages, etc.
@@ -322,6 +328,12 @@ public string BaseUrl
                             throw new ServalApiException("The client is not authenticated", status_, responseText_, headers_, null);
                         }
                         else
+                        if (status_ == 422)
+                        {
+                            string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false);
+                            throw new ServalApiException("A server side error occurred.", status_, responseText_, headers_, null);
+                        }
+                        else
                         if (status_ == 403)
                         {
                             string responseText_ = ( response_.Content == null ) ? string.Empty : await response_.Content.ReadAsStringAsync().ConfigureAwait(false);

src/Serval.DataFiles/Controllers/DataFilesController.cs

@@ -90,15 +90,19 @@ public async Task<ActionResult<DataFileDto>> GetAsync([NotNull] string id, Cance
     /// <param name="format">
     /// File format options:
     /// * **Text**: One translation unit (a.k.a., verse) per line
-    ///   * If there is a tab, the content before the tab is the unique identifier for the line
-    ///   * Otherwise, no tabs should be used in the file.
+    ///   * If a line contains a tab, characters before the tab are used as a unique identifier for the line, characters after the tab are understood as the content of the verse, and if there is another tab following the verse content, characters after this second tab are assumed to be column codes like "ss" etc. for sectioning and other formatting. See this example of a tab-delimited text file:
+    ///     > verse_001_005 (tab) Ὑπομνῆσαι δὲ ὑμᾶς βούλομαι , εἰδότας ὑμᾶς ἅπαξ τοῦτο
+    ///     > verse_001_006	(tab) Ἀγγέλους τε τοὺς μὴ τηρήσαντας τὴν ἑαυτῶν ἀρχήν , ἀλλὰ (tab) ss
+    ///     > verse_001_007 (tab) Ὡς Σόδομα καὶ Γόμορρα , καὶ αἱ περὶ αὐτὰς πόλεις (tab) ss
+    ///   * Otherwise, *no tabs* should be used in the file and a unique identifier will generated for each translation unit based on the linenumber.
     /// * **Paratext**: A complete, zipped Paratext project
     /// </param>
     /// <param name="idGenerator"></param>
     /// <param name="cancellationToken"></param>
     /// <response code="201">The file was created successfully</response>
     /// <response code="400">Bad request. Is the file over 100 MB?</response>
     /// <response code="401">The client is not authenticated</response>
+    /// <response code="402">The contents of the file were not formatted correctly</response>
     /// <response code="403">The authenticated client cannot perform the operation</response>
     /// <response code="503">A necessary service is currently unavailable. Check `/health` for more details. </response>
     [Authorize(Scopes.CreateFiles)]
@@ -107,6 +111,7 @@ public async Task<ActionResult<DataFileDto>> GetAsync([NotNull] string id, Cance
     [ProducesResponseType(StatusCodes.Status201Created)]
     [ProducesResponseType(typeof(void), StatusCodes.Status400BadRequest)]
     [ProducesResponseType(typeof(void), StatusCodes.Status401Unauthorized)]
+    [ProducesResponseType(typeof(void), StatusCodes.Status422UnprocessableEntity)]
     [ProducesResponseType(typeof(void), StatusCodes.Status403Forbidden)]
     [ProducesResponseType(typeof(void), StatusCodes.Status503ServiceUnavailable)]
     public async Task<ActionResult<DataFileDto>> CreateAsync(
@@ -126,7 +131,14 @@ CancellationToken cancellationToken
         };
         using (Stream stream = file.OpenReadStream())
         {
-            await _dataFileService.CreateAsync(dataFile, stream, cancellationToken);
+            try
+            {
+                await _dataFileService.CreateAsync(dataFile, stream, cancellationToken);
+            }
+            catch (FormatException fe)
+            {
+                return UnprocessableEntity(fe.Message);
+            }
         }
         var dto = Map(dataFile);
         return Created(dto.Url, dto);

src/Serval.DataFiles/Services/DataFileService.cs

@@ -1,4 +1,6 @@
-namespace Serval.DataFiles.Services;
+using System.Text.RegularExpressions;
+
+namespace Serval.DataFiles.Services;
 
 public class DataFileService : EntityServiceBase<DataFile>, IDataFileService
 {
@@ -38,12 +40,22 @@ public async Task<IEnumerable<DataFile>> GetAllAsync(string owner, CancellationT
 
     public async Task CreateAsync(DataFile dataFile, Stream stream, CancellationToken cancellationToken = default)
     {
+        string content = new StreamReader(stream).ReadToEnd();
+        if (
+            dataFile.Format == FileFormat.Text
+            && !Regex.IsMatch(content, @"^(((\S| )*\n)+|((\S+\t(\S| )*(\t\S+)?)\n))+$")
+        )
+        {
+            throw new FormatException(
+                "The contents of the text file were not formatted correctly. Please see the documentation and verify that your document is correctly formatted,"
+            );
+        }
         dataFile.Filename = Path.GetRandomFileName();
         string path = GetDataFilePath(dataFile.Filename);
         try
         {
             using Stream fileStream = _fileSystem.OpenWrite(path);
-            await stream.CopyToAsync(fileStream, cancellationToken);
+            await new StreamWriter(fileStream).WriteAsync(content);
             await Entities.InsertAsync(dataFile, cancellationToken);
         }
         catch

src/Serval.Translation/Controllers/TranslationEnginesController.cs

@@ -1,7 +1,4 @@
-using Serval.Translation.Models;
-using System.Threading;
-
-namespace Serval.Translation.Controllers;
+namespace Serval.Translation.Controllers;
 
 [ApiVersion(1.0)]
 [Route("api/v{version:apiVersion}/translation/engines")]