(#2860) Allow lines with incorrect column count

These used to be rejected at upload time; now they are allowed and the invalid lines are reported during data extraction.
quince-science · Feb 6, 2024 · bfe5215 · bfe5215
1 parent 8b9ffb6
commit bfe5215
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 12 deletions.
diff --git a/WebApp/src/uk/ac/exeter/QuinCe/data/Files/DataFile.java b/WebApp/src/uk/ac/exeter/QuinCe/data/Files/DataFile.java
@@ -344,29 +344,21 @@ public void validate() throws DataFileException {
             }
           }
         } catch (MissingDateTimeException | DateTimeSpecificationException e) {
-          // We don't mind bad dates in the file.
+          // We don't mind bad dates in the file -
+          // we'll just ignore those lines
         }
 
-        boolean checkColumnCount = true;
-
+        // Check the run type
         if (fileDefinition.hasRunTypes()) {
           try {
-            RunTypeCategory runType = fileDefinition.getRunTypeCategory(line);
-            if (runType.equals(RunTypeCategory.IGNORED)) {
-              checkColumnCount = false;
-            }
+            fileDefinition.getRunTypeCategory(line);
           } catch (FileDefinitionException e) {
             addMessage(lineNumber, e.getMessage());
             if (e instanceof MissingRunTypeException) {
               missingRunTypes.add(((MissingRunTypeException) e).getRunType());
             }
           }
         }
-
-        if (checkColumnCount && fileDefinition.extractFields(line)
-          .size() != fileDefinition.getColumnCount()) {
-          addMessage(lineNumber, "Incorrect number of columns");
-        }
       }
     }
   }

diff --git a/WebApp/src/uk/ac/exeter/QuinCe/jobs/files/ExtractDataSetJob.java b/WebApp/src/uk/ac/exeter/QuinCe/jobs/files/ExtractDataSetJob.java
@@ -21,7 +21,9 @@
 import uk.ac.exeter.QuinCe.data.Files.DataFile;
 import uk.ac.exeter.QuinCe.data.Files.DataFileDB;
 import uk.ac.exeter.QuinCe.data.Instrument.FileDefinition;
+import uk.ac.exeter.QuinCe.data.Instrument.FileDefinitionException;
 import uk.ac.exeter.QuinCe.data.Instrument.Instrument;
+import uk.ac.exeter.QuinCe.data.Instrument.MissingRunTypeException;
 import uk.ac.exeter.QuinCe.data.Instrument.Calibration.Calibration;
 import uk.ac.exeter.QuinCe.data.Instrument.Calibration.CalibrationSet;
 import uk.ac.exeter.QuinCe.data.Instrument.Calibration.SensorCalibrationDB;
@@ -145,6 +147,32 @@ protected void execute(JobThread thread) throws JobFailedException {
           try {
 
             List<String> line = file.getLine(currentLine);
+
+            // Check the number of columns on the line
+            boolean checkColumnCount = true;
+
+            if (fileDefinition.hasRunTypes()) {
+              try {
+                RunTypeCategory runType = fileDefinition.getRunType(line, true)
+                  .getCategory();
+                if (runType.equals(RunTypeCategory.IGNORED)) {
+                  checkColumnCount = false;
+                }
+              } catch (FileDefinitionException e) {
+                dataSet.addProcessingMessage(jobName, file, currentLine, e);
+                if (e instanceof MissingRunTypeException) {
+                  dataSet.addProcessingMessage(jobName, file, currentLine,
+                    "Unrecognised Run Type");
+                }
+              }
+            }
+
+            if (checkColumnCount
+              && line.size() != fileDefinition.getColumnCount()) {
+              dataSet.addProcessingMessage(jobName, file, currentLine,
+                "Incorrect number of columns");
+            }
+
             LocalDateTime time = file.getOffsetTime(line);
 
             if ((time.equals(dataSet.getStart())