Add GRIP campaign + code changes (#17)

NASA-IMPACT · Jun 10, 2024 · 5fa905a · 5fa905a
1 parent ccfba02
commit 5fa905a
Show file tree

Hide file tree

Showing 4 changed files with 83 additions and 1 deletion.
diff --git a/campaigns/GRIP/deployments.yaml b/campaigns/GRIP/deployments.yaml
@@ -0,0 +1,57 @@
+---
+deployments:
+- name: GRIP-D1_2010
+  platforms:
+  - name: DC-8
+    files:
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_eng_10Aug2010.csv
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight1_17Aug2010.csv
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight2_24Aug2010.csv
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight3_29Aug2010.csv
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight4_30Aug2010.csv
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight5_01Sep2010.csv
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight6_02Sep2010.csv
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight7_06Sep2010.csv
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight8_07Sep2010.csv
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight9_12Sep2010.csv
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight10_13Sep2010.csv
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight11_14Sep2010.csv
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight12_16Sep2010.csv
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight13_17Sep2010.csv
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight14_21Sep2010.csv
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight15_22Sep2010.csv
+  - name: WB-57
+    files:
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100827_WB57.dat
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100831_WB57.dat
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100901_WB57.dat
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100902_WB57.dat
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100903_WB57.dat
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100914_WB57.dat
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100915_WB57.dat
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100916_WB57.dat
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100917_WB57.dat
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.15Aug2010.xml
+  - name: GH
+    files:
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.15Aug2010-2017
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.15Aug2010.xml
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.15Aug2010-2111
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.15Aug2010-2126
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.24Aug2010.xml
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.24Aug2010-1751
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.24Aug2010-1754
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.29Aug2010.xml
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.29Aug2010-0342
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.03Sep2010-0349
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.03Sep2010.xml
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.03Sep2010-0402
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.13Sep2010-1132
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.13Sep2010.xml
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.13Sep2010-1152
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.17Sep2010.xml
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.17Sep2010-1414
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.17Sep2010-1429
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.24Sep2010.xml
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.24Sep2010-1650
+    - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.24Sep2010-1706
diff --git a/task/src/download.js b/task/src/download.js
@@ -2,7 +2,7 @@ const fs = require('fs');
 const path = require('path');
 const AdmZip = require('adm-zip');
 const download = require('download');
-const { getPlatformConfig, readCampaignYaml } = require('./utils');
+const { getPlatformConfig, readCampaignYaml, urlHasFileExtension } = require('./utils');
 
 const replaceSlash = (str) => str.replaceAll('/', '-');
 
@@ -19,6 +19,11 @@ const downloadFile = async (url, dir) => {
     zip.extractAllTo(dir);
     fs.unlinkSync(filePath);
   }
+  // The GRIP campaign has files without an extension and others with .dat that should be txt
+  if (!urlHasFileExtension(url) || url.endsWith('.dat')) {
+    const filePath = path.join(dir, path.basename(url));
+    fs.renameSync(filePath, `${filePath}.txt`);
+  }
 };
 
 const downloadCampaign = (campaignPath) => {

diff --git a/task/src/utils.js b/task/src/utils.js
@@ -64,11 +64,18 @@ const divideCoordinates = (features, coordsDivisor) => features.map((i) => {
   return { ...i, geometry: newGeometry };
 });
 
+const urlHasFileExtension = (url) => {
+  // eslint-disable-next-line no-unused-vars
+  const [name, ext] = path.basename(url).split('.');
+  return ext && [3, 4].includes(ext.length);
+};
+
 module.exports = {
   getStats,
   tsv2csv,
   concatenateFiles,
   getPlatformConfig,
   readCampaignYaml,
   divideCoordinates,
+  urlHasFileExtension,
 };
diff --git a/task/tests/utils.test.js b/task/tests/utils.test.js
@@ -4,6 +4,7 @@ const {
   getStats,
   getPlatformConfig,
   divideCoordinates,
+  urlHasFileExtension,
 } = require('../src/utils');
 
 describe('tsv2csv', () => {
@@ -30,6 +31,18 @@ describe('getPlatformConfig', () => {
   });
 });
 
+describe('urlHasFileExtension', () => {
+  it('return true if an URL ends with any 3 or 4 letters file extension', () => {
+    expect(urlHasFileExtension('https://a.com/b/c/d.ext')).toBeTruthy();
+    expect(urlHasFileExtension('https://a.com/b/c/d.WB57')).toBeTruthy();
+    expect(urlHasFileExtension('https://a.com/b/c/d.WB57')).toBeTruthy();
+    expect(urlHasFileExtension('https://a.com/b/c/d.extasd')).toBeFalsy();
+    expect(urlHasFileExtension('https://a.com/b/c/d32.ext1asd')).toBeFalsy();
+    expect(urlHasFileExtension('https://a.com/b/c/d12.ext1-asd')).toBeFalsy();
+    expect(urlHasFileExtension('https://a.com/b/c/d12.ER')).toBeFalsy();
+  });
+});
+
 describe('divideCoordinates', () => {
   it('return coordinates divided by a number', () => {
     const geojson = {