Added tests to support changes in machine

Fixes Bad corpora should be flagged #2

tests/Serval.E2ETests/ServalApiTests.cs

@@ -13,7 +13,7 @@ public void SetUp()
     }
 
     [Test]
-    public async Task GetEchoSuggestion()
+    public async Task GetEchoSuggestionAsync()
     {
         await _helperClient!.ClearEngines();
         string engineId = await _helperClient.CreateNewEngine("Echo", "es", "es", "Echo1");
@@ -25,7 +25,7 @@ public async Task GetEchoSuggestion()
     }
 
     [Test]
-    public async Task GetEchoPretranslate()
+    public async Task GetEchoPretranslateAsync()
     {
         await _helperClient!.ClearEngines();
         string engineId = await _helperClient.CreateNewEngine("Echo", "es", "es", "Echo2");
@@ -43,7 +43,7 @@ public async Task GetEchoPretranslate()
     }
 
     [Test]
-    public async Task GetSmtTranslation()
+    public async Task GetSmtTranslationAsync()
     {
         await _helperClient!.ClearEngines();
         string engineId = await _helperClient.CreateNewEngine("SmtTransfer", "es", "en", "SMT1");
@@ -56,7 +56,7 @@ public async Task GetSmtTranslation()
 
     [Test]
     [Category("slow")]
-    public async Task GetSmtWholeBible()
+    public async Task GetSmtWholeBibleAsync()
     {
         await _helperClient!.ClearEngines();
         string engineId = await _helperClient.CreateNewEngine("SmtTransfer", "es", "en", "SMT2");
@@ -67,7 +67,7 @@ public async Task GetSmtWholeBible()
     }
 
     [Test]
-    public async Task GetSmtAddSegment()
+    public async Task GetSmtAddSegmentAsync()
     {
         await _helperClient!.ClearEngines();
         string engineId = await _helperClient.CreateNewEngine("SmtTransfer", "es", "en", "SMT3");
@@ -96,7 +96,7 @@ await _helperClient.translationEnginesClient.TrainSegmentAsync(
     }
 
     [Test]
-    public async Task GetSmtMoreCorpus()
+    public async Task GetSmtMoreCorpusAsync()
     {
         await _helperClient!.ClearEngines();
         string engineId = await _helperClient.CreateNewEngine("SmtTransfer", "es", "en", "SMT4");
@@ -117,7 +117,7 @@ public async Task GetSmtMoreCorpus()
     }
 
     [Test]
-    public async Task NmtBatch()
+    public async Task NmtBatchAsync()
     {
         await _helperClient!.ClearEngines();
         string engineId = await _helperClient.CreateNewEngine("Nmt", "es", "en", "NMT1");
@@ -134,13 +134,13 @@ public async Task NmtBatch()
     }
 
     [Test]
-    public async Task GetNmtCancelAndRestartBuild()
+    public async Task GetNmtCancelAndRestartBuildAsync()
     {
         await _helperClient!.ClearEngines();
         string engineId = await _helperClient.CreateNewEngine("Nmt", "es", "en", "NMT2");
         var books = new string[] { "1JN.txt", "2JN.txt", "3JN.txt" };
         await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false);
-        await StartAndCancelTwice(engineId);
+        await StartAndCancelTwiceAsync(engineId);
     }
 
     [Test]
@@ -244,22 +244,22 @@ public async Task CircuitousRouteTranslateTopNAsync()
     }
 
     [Test]
-    public async Task GetSmtCancelAndRestartBuild()
+    public async Task GetSmtCancelAndRestartBuildAsync()
     {
         await _helperClient!.ClearEngines();
         string engineId = await _helperClient.CreateNewEngine("SmtTransfer", "es", "en", "SMT7");
         var books = new string[] { "1JN.txt", "2JN.txt", "3JN.txt" };
         await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false);
 
-        await StartAndCancelTwice(engineId);
+        await StartAndCancelTwiceAsync(engineId);
 
         // do a job normally and make sure it works.
         await _helperClient.BuildEngine(engineId);
         TranslationResult tResult = await _helperClient.translationEnginesClient.TranslateAsync(engineId, "Espíritu");
         Assert.AreEqual(tResult.Translation, "spirit");
     }
 
-    async Task StartAndCancelTwice(string engineId)
+    async Task StartAndCancelTwiceAsync(string engineId)
     {
         // start and first job
         var build = await _helperClient!.StartBuildAsync(engineId);
@@ -283,4 +283,25 @@ async Task StartAndCancelTwice(string engineId)
         build = await _helperClient.translationEnginesClient.GetBuildAsync(engineId, build.Id);
         Assert.That(build.State == JobState.Canceled);
     }
+
+    [Test]
+    [TestCase("2JN.txt", true, false)]
+    [TestCase("2JN_EXTRATAB.txt", false, false)]
+    [TestCase("2JN_MIXEDREFS.txt", false, false)]
+    [TestCase("2JN_USERDEFINEDREFS.txt", false, false)]
+    [TestCase("2JN_EXTRATAB.txt", true, true)]
+    [TestCase("2JN_MIXEDREFS.txt", true, true)]
+    [TestCase("2JN_USERDEFINEDREFS.txt", true, false)]
+    public async Task BuildWithMalformedFilesAsync(string filename, bool useStrictParsing, bool shouldFail)
+    {
+        await _helperClient!.ClearEngines();
+        string engineId = await _helperClient.CreateNewEngine("SmtTransfer", "es", "en", "SMT7");
+        var books = new string[] { filename };
+        await _helperClient.AddTextCorpusToEngine(engineId, books, "es", "en", false);
+        var buildId = await _helperClient.BuildEngine(engineId, useStrictParsing);
+        Assert.That(
+            (await _helperClient.translationEnginesClient.GetBuildAsync(engineId, buildId)).State == JobState.Faulted,
+            Is.EqualTo(shouldFail)
+        );
+    }
 }

tests/Serval.E2ETests/ServalClientHelper.cs

@@ -106,14 +106,15 @@ public async Task<string> CreateNewEngine(
         return engine.Id;
     }
 
-    public async Task<TranslationBuild> StartBuildAsync(string engineId)
+    public async Task<TranslationBuild> StartBuildAsync(string engineId, bool useStrictParsing = false)
     {
+        translationBuildConfig.Options = "{\"useStrictParsing\":" + $"\"{useStrictParsing}\"" + "}";
         return await translationEnginesClient.StartBuildAsync(engineId, translationBuildConfig);
     }
 
-    public async Task BuildEngine(string engineId)
+    public async Task<string> BuildEngine(string engineId, bool useStrictParsing = false)
     {
-        var newJob = await StartBuildAsync(engineId);
+        var newJob = await StartBuildAsync(engineId, useStrictParsing);
         await translationEnginesClient.GetBuildAsync(engineId, newJob.Id, newJob.Revision);
         int pollIntervalMs = 500; // start throttle at 0.5 seconds
         while (true)
@@ -129,6 +130,7 @@ public async Task BuildEngine(string engineId)
             // increase throttle exponentially to 10 seconds
             pollIntervalMs = (int)Math.Min(pollIntervalMs * 1.2, 10_000);
         }
+        return newJob.Id;
     }
 
     public async Task CancelBuild(string engineId, string buildId, int timeoutSeconds = 20)

tests/Serval.E2ETests/data/en/2JN_EXTRATAB.txt

@@ -0,0 +1,13 @@
+From the elder to the chosen lady and her children, whom I love in truth—and not only I, but also all those who have known the truth	
+because of the truth that remains in us and will be with us forever.
+Grace, mercy, and peace will be with us from God the Father and from Jesus Christ, the Son of the Father, in truth and love.
+I rejoice greatly that I have found some of your children walking in truth, just as we have received this commandment from the Father.
+Now I plead with you, lady—not as though I were writing to you a new commandment, but one that we have had from the beginning—that we should love one another.
+This is love, that we should walk according to his commandments. This is the commandment, just as you heard from the beginning, that you should walk in it.
+For many deceivers have gone out into the world, and they do not confess that Jesus Christ came in the flesh. This is the deceiver and the antichrist.	
+Look to yourselves, that you do not lose the things for which we all have worked, but so that you may receive a full reward.
+Whoever goes on ahead and does not remain in the teaching of Christ does not have God. The one who remains in the teaching, this one has both the Father and the Son.
+If anyone comes to you and does not bring this teaching, do not receive him into your house and do not greet him.
+For the one who greets him participates in his evil deeds.
+I have many things to write to you, but I did not wish to write them with paper and ink. However, I expect to come to you and speak face to face, so that our joy will be complete.
+The children of your chosen sister greet you.

tests/Serval.E2ETests/data/en/2JN_MIXEDREFS.txt

@@ -0,0 +1,13 @@
+r1	From the elder to the chosen lady and her children, whom I love in truth—and not only I, but also all those who have known the truth	
+because of the truth that remains in us and will be with us forever.
+Grace, mercy, and peace will be with us from God the Father and from Jesus Christ, the Son of the Father, in truth and love.
+r3	I rejoice greatly that I have found some of your children walking in truth, just as we have received this commandment from the Father.
+Now I plead with you, lady—not as though I were writing to you a new commandment, but one that we have had from the beginning—that we should love one another.
+This is love, that we should walk according to his commandments. This is the commandment, just as you heard from the beginning, that you should walk in it.
+For many deceivers have gone out into the world, and they do not confess that Jesus Christ came in the flesh. This is the deceiver and the antichrist.	
+Look to yourselves, that you do not lose the things for which we all have worked, but so that you may receive a full reward.
+Whoever goes on ahead and does not remain in the teaching of Christ does not have God. The one who remains in the teaching, this one has both the Father and the Son.
+If anyone comes to you and does not bring this teaching, do not receive him into your house and do not greet him.
+For the one who greets him participates in his evil deeds.
+I have many things to write to you, but I did not wish to write them with paper and ink. However, I expect to come to you and speak face to face, so that our joy will be complete.
+The children of your chosen sister greet you.

tests/Serval.E2ETests/data/en/2JN_USERDEFINEDREFS.txt

@@ -0,0 +1,13 @@
+r1	From the elder to the chosen lady and her children, whom I love in truth—and not only I, but also all those who have known the truth
+r2	because of the truth that remains in us and will be with us forever.
+r3	Grace, mercy, and peace will be with us from God the Father and from Jesus Christ, the Son of the Father, in truth and love.
+r4	I rejoice greatly that I have found some of your children walking in truth, just as we have received this commandment from the Father.
+r5	Now I plead with you, lady—not as though I were writing to you a new commandment, but one that we have had from the beginning—that we should love one another.
+r6	This is love, that we should walk according to his commandments. This is the commandment, just as you heard from the beginning, that you should walk in it.
+r7	For many deceivers have gone out into the world, and they do not confess that Jesus Christ came in the flesh. This is the deceiver and the antichrist.
+r8	Look to yourselves, that you do not lose the things for which we all have worked, but so that you may receive a full reward.
+r9	Whoever goes on ahead and does not remain in the teaching of Christ does not have God. The one who remains in the teaching, this one has both the Father and the Son.
+r10	If anyone comes to you and does not bring this teaching, do not receive him into your house and do not greet him.
+r11	For the one who greets him participates in his evil deeds.
+r12	I have many things to write to you, but I did not wish to write them with paper and ink. However, I expect to come to you and speak face to face, so that our joy will be complete.
+r13	The children of your chosen sister greet you.

tests/Serval.E2ETests/data/es/2JN_EXTRATAB.txt

@@ -0,0 +1,13 @@
+EL anciano á la señora elegida y á sus hijos, á los cuales yo amo en verdad; y no yo solo, sino también todos los que han conocido la verdad,
+Por la verdad que está en nosotros, y será perpetuamente con nosotros:
+Sea con vosotros gracia, misericordia, y paz de Dios Padre, y del Señor Jesucristo, Hijo del Padre, en verdad y en amor.
+Mucho me he gozado, porque he hallado de tus hijos, que andan en verdad, como nosotros hemos recibido el mandamiento del Padre.	
+Y ahora te ruego, señora, no como escribiéndote un nuevo mandamiento, sino aquel que nosotros hemos tenido desde el principio, que nos amemos unos á otros.
+Y este es amor, que andemos según sus mandamientos. Este es el mandamiento: Que andéis en él, como vosotros habéis oído desde el principio.
+Porque muchos engañadores son entrados en el mundo, los cuales no confiesan que Jesucristo ha venido en carne. Este tal el engañador es, y el anticristo.
+Mirad por vosotros mismos, porque no perdamos las cosas que hemos obrado, sino que recibamos galardón cumplido.
+Cualquiera que se rebela, y no persevera en la doctrina de Cristo, no tiene á Dios: el que persevera en la doctrina de Cristo, el tal tiene al Padre y al Hijo.
+Si alguno viene á vosotros, y no trae esta doctrina, no lo recibáis en casa, ni le digáis: ¡bienvenido!	
+Porque el que le dice bienvenido, comunica con sus malas obras.
+Aunque tengo muchas cosas que escribiros, no he querido comunicarlas por medio de papel y tinta; mas espero ir á vosotros, y hablar boca á boca, para que nuestro gozo sea cumplido.
+Los hijos de tu hermana elegida te saludan. Amén.

tests/Serval.E2ETests/data/es/2JN_MIXEDREFS.txt

@@ -0,0 +1,13 @@
+r1	EL anciano á la señora elegida y á sus hijos, á los cuales yo amo en verdad; y no yo solo, sino también todos los que han conocido la verdad,
+Por la verdad que está en nosotros, y será perpetuamente con nosotros:
+Sea con vosotros gracia, misericordia, y paz de Dios Padre, y del Señor Jesucristo, Hijo del Padre, en verdad y en amor.
+r3	Mucho me he gozado, porque he hallado de tus hijos, que andan en verdad, como nosotros hemos recibido el mandamiento del Padre.
+Y ahora te ruego, señora, no como escribiéndote un nuevo mandamiento, sino aquel que nosotros hemos tenido desde el principio, que nos amemos unos á otros.
+Y este es amor, que andemos según sus mandamientos. Este es el mandamiento: Que andéis en él, como vosotros habéis oído desde el principio.
+Porque muchos engañadores son entrados en el mundo, los cuales no confiesan que Jesucristo ha venido en carne. Este tal el engañador es, y el anticristo.
+Mirad por vosotros mismos, porque no perdamos las cosas que hemos obrado, sino que recibamos galardón cumplido.
+Cualquiera que se rebela, y no persevera en la doctrina de Cristo, no tiene á Dios: el que persevera en la doctrina de Cristo, el tal tiene al Padre y al Hijo.
+Si alguno viene á vosotros, y no trae esta doctrina, no lo recibáis en casa, ni le digáis: ¡bienvenido!
+Porque el que le dice bienvenido, comunica con sus malas obras.
+Aunque tengo muchas cosas que escribiros, no he querido comunicarlas por medio de papel y tinta; mas espero ir á vosotros, y hablar boca á boca, para que nuestro gozo sea cumplido.
+Los hijos de tu hermana elegida te saludan. Amén.

tests/Serval.E2ETests/data/es/2JN_USERDEFINEDREFS.txt

@@ -0,0 +1,13 @@
+r1	EL anciano á la señora elegida y á sus hijos, á los cuales yo amo en verdad; y no yo solo, sino también todos los que han conocido la verdad,
+r2	Por la verdad que está en nosotros, y será perpetuamente con nosotros:
+r3	Sea con vosotros gracia, misericordia, y paz de Dios Padre, y del Señor Jesucristo, Hijo del Padre, en verdad y en amor.
+r4	Mucho me he gozado, porque he hallado de tus hijos, que andan en verdad, como nosotros hemos recibido el mandamiento del Padre.
+r5	Y ahora te ruego, señora, no como escribiéndote un nuevo mandamiento, sino aquel que nosotros hemos tenido desde el principio, que nos amemos unos á otros.
+r6	Y este es amor, que andemos según sus mandamientos. Este es el mandamiento: Que andéis en él, como vosotros habéis oído desde el principio.
+r7	Porque muchos engañadores son entrados en el mundo, los cuales no confiesan que Jesucristo ha venido en carne. Este tal el engañador es, y el anticristo.
+r8	Mirad por vosotros mismos, porque no perdamos las cosas que hemos obrado, sino que recibamos galardón cumplido.
+r9	Cualquiera que se rebela, y no persevera en la doctrina de Cristo, no tiene á Dios: el que persevera en la doctrina de Cristo, el tal tiene al Padre y al Hijo.
+r10	Si alguno viene á vosotros, y no trae esta doctrina, no lo recibáis en casa, ni le digáis: ¡bienvenido!
+r11	Porque el que le dice bienvenido, comunica con sus malas obras.
+r12	Aunque tengo muchas cosas que escribiros, no he querido comunicarlas por medio de papel y tinta; mas espero ir á vosotros, y hablar boca á boca, para que nuestro gozo sea cumplido.
+r13	Los hijos de tu hermana elegida te saludan. Amén.