Add new campaigns + Update YAML schema with header_content field (#28)

NASA-IMPACT · Oct 30, 2024 · 4c7ecfe · 4c7ecfe
1 parent 13ace36
commit 4c7ecfe
Show file tree

Hide file tree

Showing 6 changed files with 122 additions and 8 deletions.
diff --git a/campaigns-schema.yaml b/campaigns-schema.yaml
@@ -8,6 +8,7 @@ deployments:
       name:
         required: true
         type: string
+        description: Deployment identifier
       platforms:
         required: true
         type: list
@@ -17,21 +18,33 @@ deployments:
             name:
               required: true
               type: string
+              description: Platform name
             tsv_format:
               required: false
               type: boolean
+              description: Whether the files are in TSV format
             rename_as_ict:
               required: false
-              type: boolean
+              description: Whether the files are ICARTT, but have another file extension. If true, they will be renamed to .ict after the download.
             process_as_ict:
               required: false
               type: boolean
+              description: Whether a .txt file should be processed as a .ict file.
             coords_divisor:
               required: false
               type: integer
+              description: It will divide the coordinates values by an integer number.
             data_start_line_fix:
               required: false
               type: integer
+              description: Some ICARTT files have a wrong data start line. This field value will be summed to the ICARTT start line to fix the error.
+            header_content:
+              required: false
+              type: string
+              description: Set a new value to the ICARTT header, as some files don't have a header.
             files:
               required: true
-              type: list
+              type: list
+              items:
+                type: string
+                description: URL of the file
diff --git a/campaigns/SOLVE/deployments.yaml b/campaigns/SOLVE/deployments.yaml
@@ -0,0 +1,56 @@
+---
+deployments:
+- name: SOLVE-D1_1999
+  platforms:
+  - name: DC-8
+    coords_divisor: 1000
+    tsv_format: true
+    rename_as_ict: true
+    files:
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/11/SOLVE-DG19991116.DC8
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/11/SOLVE-DG19991119.DC8
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/11/SOLVE-DG19991122.DC8
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/11/SOLVE-DG19991130.DC8
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/12/SOLVE-DG19991202.DC8
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/12/SOLVE-DG19991205.DC8
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/12/SOLVE-DG19991207.DC8
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/12/SOLVE-DG19991210.DC8
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/12/SOLVE-DG19991212.DC8
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/12/SOLVE-DG19991214.DC8
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/12/SOLVE-DG19991216.DC8
+  - name: ER-2
+    coords_divisor: 1000
+    tsv_format: true
+    rename_as_ict: true
+    files:
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/1999/12/SOLVE-FP19991211.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/1999/12/SOLVE-FP19991215.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/1999/12/SOLVE-FP19991216.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000106.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000109.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000111.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000114.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000120.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000123.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000127.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000131.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/02/SOLVE-FP20000202.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/02/SOLVE-FP20000203.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/02/SOLVE-FP20000226.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/03/SOLVE-FP20000305.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/03/SOLVE-FP20000307.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/03/SOLVE-FP20000311.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/03/SOLVE-FP20000312.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/03/SOLVE-FP20000316.ER2
+    - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/03/SOLVE-FP20000318.ER2
+- name: SOLVE-D2_2002
+  platforms:
+  - name: DC-8
+    coords_divisor: 10000
+    tsv_format: true
+    rename_as_ict: true
+    header_content: a,b,latitude,longitude,speed
+    files:
+    - https://asdc.larc.nasa.gov/data/SOLVE_II/MetNav_AircraftInSitu_DC8_Data_1/2002/12/SOLVE-ID20021213__FLT.DC8
+    - https://asdc.larc.nasa.gov/data/SOLVE_II/MetNav_AircraftInSitu_DC8_Data_1/2002/12/SOLVE-ID20021217__FLT.DC8
+    - https://asdc.larc.nasa.gov/data/SOLVE_II/MetNav_AircraftInSitu_DC8_Data_1/2002/12/SOLVE-ID20021219__FLT.DC8
diff --git a/campaigns/SONEX/deployments.yaml b/campaigns/SONEX/deployments.yaml
@@ -0,0 +1,25 @@
+---
+deployments:
+- name: SONEX-D1_1997
+  platforms:
+  - name: DC-8
+    coords_divisor: 1000
+    tsv_format: true
+    rename_as_ict: true
+    files:
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971007.DC8
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971009.DC8
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971013.DC8
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971015.DC8
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971018.DC8
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971020.DC8
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971023.DC8
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971025.DC8
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971028.DC8
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971029.DC8
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971031.DC8
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/11/SONEX-FP19971103.DC8
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/11/SONEX-FP19971105.DC8
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/11/SONEX-FP19971109.DC8
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/11/SONEX-FP19971110.DC8
+    - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/11/SONEX-FP19971112.DC8
diff --git a/task/makeCSV.js b/task/makeCSV.js
@@ -18,13 +18,23 @@ const makeCSV = (platformPath) => {
   // process .ict files
   const ictFiles = findFiles(platformPath, '.ict');
   ictFiles.forEach(
-    (f) => splitICTFile(f, platformConfig.tsv_format, platformConfig.data_start_line_fix)
+    (f) => splitICTFile(
+      f,
+      platformConfig.tsv_format,
+      platformConfig.data_start_line_fix,
+      platformConfig.header_content
+    )
   );
 
   // some platforms have txt files formatted as icartt
   if (platformConfig.process_as_ict) {
     txtFiles.forEach(
-      (f) => splitICTFile(f, platformConfig.tsv_format, platformConfig.data_start_line_fix)
+      (f) => splitICTFile(
+        f,
+        platformConfig.tsv_format,
+        platformConfig.data_start_line_fix,
+        platformConfig.header_content
+      )
     );
   }
 };

diff --git a/task/src/process.js b/task/src/process.js
@@ -31,17 +31,21 @@ const getPropertiesFromPath = (dir) => {
 * @param {Number} dataStartLineFix - sum or substract a value to/from the dataStartLine
 * value in the ict header
 */
-const getDataContentFromICT = (filename, dataStartLineFix = 0) => {
+const getDataContentFromICT = (filename, dataStartLineFix = 0, replaceHeaderContent = '') => {
   const file = fs.readFileSync(filename);
   let content = file.toString().split('\n');
   let dataStartLine;
+  let header;
   // some campaigns have a different header identifier
   if (content[0] === '/begin_header') {
     dataStartLine = content.indexOf('/end_header');
-    const header = content[dataStartLine - 2].replace('/fields=', '');
+    header = content[dataStartLine - 2].replace('/fields=', '');
     return [header, ...content.slice(dataStartLine + 1)].join('\n');
   }
   dataStartLine = Number(content[0].split(/\s*,?\s*1001/g)[0]);
+  if (replaceHeaderContent) {
+    return [replaceHeaderContent, ...content.slice(dataStartLine - 1)].join('\n');
+  }
   content = content.slice(dataStartLine - 1 + dataStartLineFix).join('\n');
   return content;
 };
@@ -53,8 +57,13 @@ const getDataContentFromICT = (filename, dataStartLineFix = 0) => {
 * @param {Number} dataStartLineFix - sum or substract a value to/from the dataStartLine
 * value in the ict header
 */
-const splitICTFile = (filename, isTSVFormatted = false, dataStartLineFix = 0) => {
-  let content = getDataContentFromICT(filename, dataStartLineFix);
+const splitICTFile = (
+  filename,
+  isTSVFormatted = false,
+  dataStartLineFix = 0,
+  replaceHeaderContent = ''
+) => {
+  let content = getDataContentFromICT(filename, dataStartLineFix, replaceHeaderContent);
 
   if (isTSVFormatted) {
     content = tsv2csv(content);

diff --git a/task/tests/validate-yaml.test.js b/task/tests/validate-yaml.test.js
@@ -22,6 +22,7 @@ const structure = array(
         process_as_ict: optional(boolean()),
         coords_divisor: optional(number()),
         data_start_line_fix: optional(number()),
+        header_content: optional(string()),
         files: array(string()),
       })
     ),