Add new campaings and improve ICT split method

NASA-IMPACT · Aug 16, 2024 · 6164745 · 6164745
1 parent e1c5079
commit 6164745
Show file tree

Hide file tree

Showing 4 changed files with 224 additions and 24 deletions.
diff --git a/campaigns/AAOE/deployments.yaml b/campaigns/AAOE/deployments.yaml
@@ -0,0 +1,46 @@
+---
+deployments:
+- name: AAOE-D1_1987
+  platforms:
+  - name: ER-2
+    tsv_format: true
+    rename_as_ict: true
+    coords_divisor: 1000
+    files:
+    - https://espoarchive.nasa.gov/archive/download/FP19870812.ER2
+    - https://espoarchive.nasa.gov/archive/download/FP19870814.ER2
+    - https://espoarchive.nasa.gov/archive/download/FP19870815.ER2
+    - https://espoarchive.nasa.gov/archive/download/FP19870817.ER2
+    - https://espoarchive.nasa.gov/archive/download/FP19870828.ER2
+    - https://espoarchive.nasa.gov/archive/download/FP19870830.ER2
+    - https://espoarchive.nasa.gov/archive/download/FP19870902.ER2
+    - https://espoarchive.nasa.gov/archive/download/FP19870904.ER2
+    - https://espoarchive.nasa.gov/archive/download/FP19870909.ER2
+    - https://espoarchive.nasa.gov/archive/download/FP19870916.ER2
+    - https://espoarchive.nasa.gov/archive/download/FP19870920.ER2
+    - https://espoarchive.nasa.gov/archive/download/FP19870921.ER2
+    - https://espoarchive.nasa.gov/archive/download/FP19870922.ER2
+    - https://espoarchive.nasa.gov/archive/download/FP19870929.ER2
+    - https://espoarchive.nasa.gov/archive/download/FP19871003.ER2
+  - name: DC-8
+    tsv_format: true
+    rename_as_ict: true
+    coords_divisor: 1000
+    files:
+    - https://espoarchive.nasa.gov/archive/download/NP19870819.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19870822.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19870828.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19870830.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19870902.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19870905.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19870908.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19870911.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19870914.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19870916.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19870919.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19870921.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19870924.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19870926.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19870929.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19871002.DC8
+    - https://espoarchive.nasa.gov/archive/download/NP19871004.DC8
diff --git a/campaigns/AASE/deployments.yaml b/campaigns/AASE/deployments.yaml
@@ -0,0 +1,109 @@
+---
+deployments:
+- name: AASE-D1_1988-Dec
+  platforms:
+  - name: ER-2
+    tsv_format: true
+    rename_as_ict: true
+    coords_divisor: 1000
+    files:
+    - https://espo.nasa.gov/aase/archive/download/FP19881229.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19881231.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890103.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890106.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890107.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890112.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890116.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890119.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890120.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890124.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890125.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890130.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890207.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890208.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890209.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890210.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890220.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19890221.ER2
+  - name: DC-8
+    tsv_format: true
+    rename_as_ict: true
+    coords_divisor: 1000
+    files:
+    - https://espo.nasa.gov/aase/archive/download/NP19890102.DC8
+    - https://espo.nasa.gov/aase/archive/download/NP19890106.DC8
+    - https://espo.nasa.gov/aase/archive/download/NP19890109.DC8
+    - https://espo.nasa.gov/aase/archive/download/NP19890111.DC8
+    - https://espo.nasa.gov/aase/archive/download/NP19890114.DC8
+    - https://espo.nasa.gov/aase/archive/download/NP19890117.DC8
+    - https://espo.nasa.gov/aase/archive/download/NP19890119.DC8
+    - https://espo.nasa.gov/aase/archive/download/NP19890124.DC8
+    - https://espo.nasa.gov/aase/archive/download/NP19890126.DC8
+    - https://espo.nasa.gov/aase/archive/download/NP19890129.DC8
+    - https://espo.nasa.gov/aase/archive/download/NP19890131.DC8
+    - https://espo.nasa.gov/aase/archive/download/NP19890202.DC8
+    - https://espo.nasa.gov/aase/archive/download/NP19890205.DC8
+    - https://espo.nasa.gov/aase/archive/download/NP19890207.DC8
+    - https://espo.nasa.gov/aase/archive/download/NP19890209.DC8
+    - https://espo.nasa.gov/aase/archive/download/NP19890215.DC8
+- name: AASE-D2_1991-Sep
+  platforms:
+  - name: ER-2
+    tsv_format: true
+    rename_as_ict: true
+    coords_divisor: 1000
+    files:
+    - https://espo.nasa.gov/aase/archive/download/FP19910917.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19910919.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19911004.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19911006.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19911008.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19911010.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19911012.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19911014.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19911102.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19911104.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19911208.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19911210.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19911212.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19911214.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920104.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920106.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920108.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920111.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920112.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920116.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920120.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920213.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920215.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920217.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920222.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920315.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920318.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920320.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920322.ER2
+    - https://espo.nasa.gov/aase/archive/download/FP19920326.ER2
+  - name: DC-8
+    tsv_format: true
+    rename_as_ict: true
+    coords_divisor: 1000
+    files:
+    - https://espo.nasa.gov/aase/archive/download/DF19920108.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920111.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920114.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920116.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920119.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920122.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920123.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920128.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920129.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920212.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920214.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920217.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920220.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920222.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920310.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920312.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920314.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920318.DC8
+    - https://espo.nasa.gov/aase/archive/download/DF19920320.DC8
diff --git a/campaigns/ATom/deployments.yaml b/campaigns/ATom/deployments.yaml
@@ -0,0 +1,65 @@
+---
+deployments:
+- name: ATom-D1_2016
+  platforms:
+  - name: DC-8
+    coords_divisor: 10000
+    files:
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160823_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160822_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160820_R2.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160817_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160815_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160812_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160808_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160806_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160803_R3.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160801_R2.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160729_R1.ict
+- name: ATom-D2_2017-Jan
+  platforms:
+  - name: DC-8
+    coords_divisor: 10000
+    files:
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170221_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170219_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170218_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170215_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170213_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170210_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170205_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170203_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170201_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170129_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170126_R1.ict
+- name: ATom-D3_2017-Sep
+  platforms:
+  - name: DC-8
+    coords_divisor: 10000
+    files:
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20171017_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20171014_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20171011_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20171008_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20171006_R2.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20171004_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20171001_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170928_R1.ict
+- name: ATom-D4_2018
+  platforms:
+  - name: DC-8
+    coords_divisor: 100000
+    files:
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180521_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180519_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180518_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180517_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180514_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180512_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180509_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180506_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180503_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180501_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180429_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180427_R1.ict
+    - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180424_R1.ict
diff --git a/task/src/process.js b/task/src/process.js
@@ -30,41 +30,21 @@ const getPropertiesFromPath = (dir) => {
 */
 const splitICTFile = (filename, isTSVFormatted = false) => {
   const file = fs.readFileSync(filename);
-  let content = file.toString();
+  let content = file.toString().split('\n');
+  const dataStartLine = Number(content[0].split(/\s*,?\s*1001/g)[0]);
+  content = content.slice(dataStartLine - 1).join('\n');
   if (isTSVFormatted) {
     content = tsv2csv(content);
   }
 
-  // ICART files can have different column names for the start time
-  const possibleFirstColumnNames = [
-    'index, time_start,',
-    'Gps_time,',
-    'Time_Start,',
-    'TIME_START,',
-    'StartTime_UTsec,',
-    'Time_mid,',
-    'Gps_time_midpoint,',
-    'Start_UTC,',
-    'UTC,',
-    'Time',
-    'TIME_NP,',
-    'UT',
-  ];
-  let columnNotFound = true;
-  possibleFirstColumnNames.forEach((col) => {
-    if (content.indexOf(col) !== -1 && columnNotFound) {
-      content = content.substr(content.lastIndexOf(col));
-      columnNotFound = false;
-    }
-  });
-
   // some files have different column names for latitude and longitude
   content = content
     .replace(',Lat,', ',latitude,')
     .replace(',Long,', ',longitude,')
     .replace(',Lon,', ',longitude,')
     .replace(', LAT,', ',latitude,')
     .replace(', LONG,', ',longitude,')
+    .replace(',LONG,', ',longitude,')
     .replace(',LAT,', ',latitude,')
     .replace(',LON,', ',longitude,')
     .replace(',FMS_LAT,', ',latitude,')