Add ARCTAS campaign and change code to download and process it

NASA-IMPACT · May 20, 2024 · 1896d64 · 1896d64
1 parent e9c8727
commit 1896d64
Show file tree

Hide file tree

Showing 6 changed files with 95 additions and 4 deletions.
diff --git a/campaigns/ARCTAS/deployments.yaml b/campaigns/ARCTAS/deployments.yaml
@@ -0,0 +1,71 @@
+---
+deployments:
+- name: ARCTAS-D1_2008-Apr
+  platforms:
+  - name: DC-8
+    tsv_format: true
+    files:
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080401_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080404_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080405_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080408_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080409_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080412_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080416_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080417_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080419_R1.zip
+  - name: P-3
+    tsv_format: true
+    files:
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080331_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080401_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080406_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080408_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080409_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080413_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080415_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080419_R2.zip
+- name: ARCTAS-D2_2008-Jun_California
+  platforms:
+  - name: DC-8
+    tsv_format: true
+    files:
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080624_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080622_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080620_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080618_R1.zip
+  - name: P-3
+    tsv_format: true
+    files:
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080622_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080624_R2.zip
+- name: ARCTAS-D3_2008-Jun
+  platforms:
+  - name: DC-8
+    tsv_format: true
+    files:
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080626_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080629_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080701_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080704_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080705_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080708_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080709_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080710_R1.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080713_R1.zip
+  - name: P-3
+    tsv_format: true
+    files:
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080626_R2_FLT13.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080626_R2_FLT14.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080628_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080629_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080630_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080702_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080703_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080706_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080707_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080709_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080710_R2.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080712_R2_FLT24.zip
+    - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080712_R2_FLT25.zip
diff --git a/campaigns/ARCTAS/static.csv b/campaigns/ARCTAS/static.csv
@@ -11,7 +11,7 @@ ARCTAS,ARCTAS-D1_2008-Apr,Balloon Launch Site,44.95,-59.92
 ARCTAS,ARCTAS-D1_2008-Apr,Balloon Launch Site,53.55,-114.11
 ARCTAS,ARCTAS-D1_2008-Apr,Balloon Launch Site,60.7,-135.07
 ARCTAS,ARCTAS-D1_2008-Apr,Balloon Launch Site,41.4,-71.5
-ARCTAS,ARCTAS-D1_2008-Apr,Balloon Launch Site,71.32,156.6
+ARCTAS,ARCTAS-D1_2008-Apr,Balloon Launch Site,71.32,-156.6
 ARCTAS,ARCTAS-D2_2008-Jun,Field Site,71.3,-156.74
 ARCTAS,ARCTAS-D2_2008-Jun,Balloon Launch Site,41.4,-71.5
 ARCTAS,ARCTAS-D2_2008-Jun,Balloon Launch Site,39.95,-105.25

diff --git a/task/package.json b/task/package.json
@@ -13,6 +13,7 @@
   "dependencies": {
     "@mapbox/geojson-merge": "^1.1.1",
     "@turf/distance": "^6.5.0",
+    "adm-zip": "^0.5.12",
     "csv2geojson": "^5.1.2",
     "d3-dsv": "1.0.1",
     "download": "8.0.0",

diff --git a/task/src/download.js b/task/src/download.js
@@ -1,6 +1,6 @@
 const fs = require('fs');
 const path = require('path');
-
+const AdmZip = require('adm-zip');
 const download = require('download');
 const { getPlatformConfig, readCampaignYaml } = require('./utils');
 
@@ -12,6 +12,13 @@ const downloadFile = async (url, dir) => {
     dir,
     { headers: { Authorization: `Bearer ${process.env.EARTH_DATA_TOKEN}` } }
   );
+  // if the file is a zip, decompress it
+  if (url.endsWith('.zip')) {
+    const filePath = path.join(dir, path.basename(url));
+    const zip = new AdmZip(filePath);
+    zip.extractAllTo(dir);
+    fs.unlinkSync(filePath);
+  }
 };
 
 const downloadCampaign = (campaignPath) => {

diff --git a/task/src/process.js b/task/src/process.js
@@ -33,12 +33,12 @@ const splitICTFile = (filename, isTSVFormatted = false) => {
   if (isTSVFormatted) {
     content = tsv2csv(content);
   }
-  let columnNotFound = true;
 
   // ICART files can have different column names for the start time
   const possibleFirstColumnNames = [
-    'Time_Start,', 'Time_mid,', 'Start_UTC,', 'TIME_NP,', 'UTC,', 'Time', 'UT',
+    'Time_Start,', 'Time_mid,', 'Time', 'UTC,', 'Start_UTC,', 'TIME_NP,', 'UT',
   ];
+  let columnNotFound = true;
   possibleFirstColumnNames.forEach((col) => {
     if (content.indexOf(col) !== -1 && columnNotFound) {
       content = content.substr(content.lastIndexOf(col));
@@ -53,6 +53,10 @@ const splitICTFile = (filename, isTSVFormatted = false) => {
     .replace(',Lon,', ',longitude,')
     .replace(', LAT,', ',latitude,')
     .replace(', LONG,', ',longitude,')
+    .replace(',LAT,', ',latitude,')
+    .replace(',LON,', ',longitude,')
+    .replace(',FMS_LAT,', ',latitude,')
+    .replace(',FMS_LON,', ',longitude,')
     .replace(',GGLAT,', ',latitude,')
     .replace(',GGLON,', ',longitude,')
     .replace(',GLAT,', ',latitude,')
@@ -68,6 +72,9 @@ const splitICTFile = (filename, isTSVFormatted = false) => {
     .replace(', Latitude,', ',latitude,')
     .replace(', Longitude,', ',longitude,');
 
+  const header = content.substr(0, content.indexOf('\n')).replaceAll(',,', ',');
+  content = `${header}${content.substr(content.indexOf('\n'))}`;
+
   const newFileName = filename.endsWith('.ict')
     ? filename.replace('.ict', '.csv')
     : `${filename}.csv`;

diff --git a/task/yarn.lock b/task/yarn.lock
@@ -820,6 +820,11 @@ adler-32@~1.3.0:
   resolved "https://registry.yarnpkg.com/adler-32/-/adler-32-1.3.1.tgz#1dbf0b36dda0012189a32b3679061932df1821e2"
   integrity sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==
 
+adm-zip@^0.5.12:
+  version "0.5.12"
+  resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.12.tgz#87786328e91d54b37358d8a50f954c4cd73ba60b"
+  integrity sha512-6TVU49mK6KZb4qG6xWaaM4C7sA/sgUMLy/JYMOzkcp3BvVLpW0fXDFQiIzAuxFCt/2+xD7fNIiPFAoLZPhVNLQ==
+
 ajv@^6.12.4:
   version "6.12.6"
   resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"