Add TCSP campaign + handle tar files

NASA-IMPACT · May 24, 2024 · e92149e · e92149e
1 parent ccfba02
commit e92149e
Show file tree

Hide file tree

Showing 6 changed files with 263 additions and 8 deletions.
diff --git a/campaigns/TCSP/deployments.yaml b/campaigns/TCSP/deployments.yaml
@@ -0,0 +1,30 @@
+---
+deployments:
+- name: TCSP_D1_2005
+  platforms:
+  - name: ER-2
+    process_as_ict: true
+    tsv_format: true
+    files:
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcsper2nav__1/tcsp_naver2_20050702_9024.txt
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcsper2nav__1/tcsp_naver2_20050705_9025.txt
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcsper2nav__1/tcsp_naver2_20050706_9026.txt
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcsper2nav__1/tcsp_naver2_20050709_9027.txt
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcsper2nav__1/tcsp_naver2_20050715_9028.txt
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcsper2nav__1/tcsp_naver2_20050716_9029.txt
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcsper2nav__1/tcsp_naver2_20050717_9030.txt
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcsper2nav__1/tcsp_naver2_20050720_9031.txt
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcsper2nav__1/tcsp_naver2_20050723_9032.txt
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcsper2nav__1/tcsp_naver2_20050724_9033.txt
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcsper2nav__1/tcsp_naver2_20050725_9034.txt
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcsper2nav__1/tcsp_naver2_20050727_9035.txt
+  - name: Aerosonde
+    process_as_ict: true
+    files:
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcspaero__1/aerosonde_2005.202.tar
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcspaero__1/aerosonde_2005.199.tar
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcspaero__1/aerosonde_2005.195.tar
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcspaero__1/aerosonde_2005.194.tar
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcspaero__1/aerosonde_2005.192.tar
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcspaero__1/aerosonde_2005.191.tar
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/tcspaero__1/aerosonde_2005.188.tar
diff --git a/task/package.json b/task/package.json
@@ -23,6 +23,7 @@
     "simplify-geojson": "^1.0.5",
     "slugify": "^1.6.6",
     "superstruct": "^1.0.4",
+    "tar": "^7.1.0",
     "xlsx": "^0.18.5",
     "xml2js": "^0.6.2",
     "yaml": "2.3.4"

diff --git a/task/src/download.js b/task/src/download.js
@@ -2,7 +2,7 @@ const fs = require('fs');
 const path = require('path');
 const AdmZip = require('adm-zip');
 const download = require('download');
-const { getPlatformConfig, readCampaignYaml } = require('./utils');
+const { getPlatformConfig, readCampaignYaml, extractFromTar } = require('./utils');
 
 const replaceSlash = (str) => str.replaceAll('/', '-');
 
@@ -12,13 +12,18 @@ const downloadFile = async (url, dir) => {
     dir,
     { headers: { Authorization: `Bearer ${process.env.EARTH_DATA_TOKEN}` } }
   );
-  // if the file is a zip, decompress it
+  // if the file is a zip or tar, decompress it
   if (url.endsWith('.zip')) {
     const filePath = path.join(dir, path.basename(url));
     const zip = new AdmZip(filePath);
     zip.extractAllTo(dir);
     fs.unlinkSync(filePath);
   }
+  if (url.endsWith('.tar')) {
+    const filePath = path.join(dir, path.basename(url));
+    await extractFromTar(filePath, dir, '.txt');
+    fs.unlinkSync(filePath);
+  }
 };
 
 const downloadCampaign = (campaignPath) => {

diff --git a/task/src/process.js b/task/src/process.js
@@ -36,7 +36,16 @@ const splitICTFile = (filename, isTSVFormatted = false) => {
 
   // ICART files can have different column names for the start time
   const possibleFirstColumnNames = [
-    'index, time_start,', 'Time_Start,', 'StartTime_UTsec,', 'Time_mid,', 'Time', 'UTC,', 'Start_UTC,', 'TIME_NP,', 'UT',
+    'index, time_start,',
+    'Time_Start,',
+    'StartTime_UTsec,',
+    'Time_mid,',
+    'Time',
+    'UTC,',
+    'Start_UTC,',
+    'TIME_NP,',
+    'UT',
+    'day:hh:mm:ss',
   ];
   let columnNotFound = true;
   possibleFirstColumnNames.forEach((col) => {
@@ -50,6 +59,8 @@ const splitICTFile = (filename, isTSVFormatted = false) => {
   content = content
     .replace(',Lat,', ',latitude,')
     .replace(',Long,', ',longitude,')
+    .replace(',lat,', ',latitude,')
+    .replace(',lon,', ',longitude,')
     .replace(',Lon,', ',longitude,')
     .replace(', LAT,', ',latitude,')
     .replace(', LONG,', ',longitude,')
@@ -71,6 +82,8 @@ const splitICTFile = (filename, isTSVFormatted = false) => {
     .replace(',POS_Lon,', ',longitude,')
     .replace(', Latitude,', ',latitude,')
     .replace(', Longitude,', ',longitude,')
+    .replace(',iLatitude,', ',latitude,')
+    .replace(',iLongitude,', ',longitude,')
     .replace(', ship_log_interp_lat,', ',latitude,')
     .replace(', ship_log_interp_lon,', ',longitude,');
 

diff --git a/task/src/utils.js b/task/src/utils.js
@@ -1,5 +1,6 @@
 const fs = require('fs');
 const path = require('path');
+const tar = require('tar');
 const { parse } = require('yaml');
 
 const getMax = (arr) => {
@@ -28,7 +29,16 @@ const getStats = (arr) => ({
   avg: arr.reduce((a, b) => a + b, 0) / arr.length,
 });
 
-const tsv2csv = (tsvContent) => tsvContent.replace(/( |\t)+/g, ',').replace(/^,+/gm, '');
+const tsv2csv = (tsvContent) => tsvContent
+  // replace "G 123:" with "123:". 123 can be any number between 1 and 3 digits
+  .replace(/G\s(\d{1,3}):/g, '$1:')
+  // replace " N 1" or " E 1" with " 1"
+  .replaceAll(/\s([NE])\s?(\d{1,3})./g, ' $2.')
+  // replace " W 1" or " S 1" with " -1"
+  .replaceAll(/\s([WS])\s?(\d{1,3})./g, ' -$2.')
+  // replace whitespaces with commas
+  .replace(/( |\t)+/g, ',')
+  .replace(/^,+/gm, '');
 
 const concatenateFiles = (file1, file2, output) => {
   const data1 = fs.readFileSync(file1);
@@ -64,11 +74,31 @@ const divideCoordinates = (features, coordsDivisor) => features.map((i) => {
   return { ...i, geometry: newGeometry };
 });
 
+const extractFromTar = async (tarFilePath, destination, extension) => {
+  await tar.list({
+    file: tarFilePath,
+    onentry: (entry) => {
+      if (entry.path.endsWith(extension)) {
+        entry.pipe(fs.createWriteStream(`${destination}/${entry.path}`));
+      } else {
+        entry.resume(); // Skip non-.txt files
+      }
+    },
+  }, (err) => {
+    if (err) {
+      console.error('Error extracting tar file:', err);
+    } else {
+      console.log('Tar file extracted successfully!');
+    }
+  });
+};
+
 module.exports = {
   getStats,
   tsv2csv,
   concatenateFiles,
   getPlatformConfig,
   readCampaignYaml,
   divideCoordinates,
+  extractFromTar,
 };