From 5fa905afe172e58d108943c57bbf19f22d4e315c Mon Sep 17 00:00:00 2001 From: Wille Marcel Date: Sun, 9 Jun 2024 19:36:34 -0600 Subject: [PATCH] Add GRIP campaign + code changes (#17) --- campaigns/GRIP/deployments.yaml | 57 +++++++++++++++++++++++++++++++++ task/src/download.js | 7 +++- task/src/utils.js | 7 ++++ task/tests/utils.test.js | 13 ++++++++ 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 campaigns/GRIP/deployments.yaml diff --git a/campaigns/GRIP/deployments.yaml b/campaigns/GRIP/deployments.yaml new file mode 100644 index 0000000..5e6b740 --- /dev/null +++ b/campaigns/GRIP/deployments.yaml @@ -0,0 +1,57 @@ +--- +deployments: +- name: GRIP-D1_2010 + platforms: + - name: DC-8 + files: + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_eng_10Aug2010.csv + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight1_17Aug2010.csv + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight2_24Aug2010.csv + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight3_29Aug2010.csv + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight4_30Aug2010.csv + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight5_01Sep2010.csv + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight6_02Sep2010.csv + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight7_06Sep2010.csv + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight8_07Sep2010.csv + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight9_12Sep2010.csv + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight10_13Sep2010.csv + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight11_14Sep2010.csv + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight12_16Sep2010.csv + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight13_17Sep2010.csv + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight14_21Sep2010.csv + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavdc8__1/GRIP_DC8_QC_NAV_flight15_22Sep2010.csv + - name: WB-57 + files: + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100827_WB57.dat + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100831_WB57.dat + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100901_WB57.dat + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100902_WB57.dat + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100903_WB57.dat + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100914_WB57.dat + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100915_WB57.dat + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100916_WB57.dat + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavwb57__1/GRIP_20100917_WB57.dat + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.15Aug2010.xml + - name: GH + files: + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.15Aug2010-2017 + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.15Aug2010.xml + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.15Aug2010-2111 + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.15Aug2010-2126 + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.24Aug2010.xml + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.24Aug2010-1751 + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.24Aug2010-1754 + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.29Aug2010.xml + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.29Aug2010-0342 + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.03Sep2010-0349 + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.03Sep2010.xml + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.03Sep2010-0402 + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.13Sep2010-1132 + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.13Sep2010.xml + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.13Sep2010-1152 + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.17Sep2010.xml + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.17Sep2010-1414 + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.17Sep2010-1429 + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.24Sep2010.xml + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.24Sep2010-1650 + - https://data.ghrc.earthdata.nasa.gov/ghrcw-protected/gripnavgh__1/IWG1.24Sep2010-1706 \ No newline at end of file diff --git a/task/src/download.js b/task/src/download.js index 2c74e8d..267c44c 100644 --- a/task/src/download.js +++ b/task/src/download.js @@ -2,7 +2,7 @@ const fs = require('fs'); const path = require('path'); const AdmZip = require('adm-zip'); const download = require('download'); -const { getPlatformConfig, readCampaignYaml } = require('./utils'); +const { getPlatformConfig, readCampaignYaml, urlHasFileExtension } = require('./utils'); const replaceSlash = (str) => str.replaceAll('/', '-'); @@ -19,6 +19,11 @@ const downloadFile = async (url, dir) => { zip.extractAllTo(dir); fs.unlinkSync(filePath); } + // The GRIP campaign has files without an extension and others with .dat that should be txt + if (!urlHasFileExtension(url) || url.endsWith('.dat')) { + const filePath = path.join(dir, path.basename(url)); + fs.renameSync(filePath, `${filePath}.txt`); + } }; const downloadCampaign = (campaignPath) => { diff --git a/task/src/utils.js b/task/src/utils.js index c91c42f..4f2ece4 100644 --- a/task/src/utils.js +++ b/task/src/utils.js @@ -64,6 +64,12 @@ const divideCoordinates = (features, coordsDivisor) => features.map((i) => { return { ...i, geometry: newGeometry }; }); +const urlHasFileExtension = (url) => { + // eslint-disable-next-line no-unused-vars + const [name, ext] = path.basename(url).split('.'); + return ext && [3, 4].includes(ext.length); +}; + module.exports = { getStats, tsv2csv, @@ -71,4 +77,5 @@ module.exports = { getPlatformConfig, readCampaignYaml, divideCoordinates, + urlHasFileExtension, }; diff --git a/task/tests/utils.test.js b/task/tests/utils.test.js index 682325d..a2bc5ff 100644 --- a/task/tests/utils.test.js +++ b/task/tests/utils.test.js @@ -4,6 +4,7 @@ const { getStats, getPlatformConfig, divideCoordinates, + urlHasFileExtension, } = require('../src/utils'); describe('tsv2csv', () => { @@ -30,6 +31,18 @@ describe('getPlatformConfig', () => { }); }); +describe('urlHasFileExtension', () => { + it('return true if an URL ends with any 3 or 4 letters file extension', () => { + expect(urlHasFileExtension('https://a.com/b/c/d.ext')).toBeTruthy(); + expect(urlHasFileExtension('https://a.com/b/c/d.WB57')).toBeTruthy(); + expect(urlHasFileExtension('https://a.com/b/c/d.WB57')).toBeTruthy(); + expect(urlHasFileExtension('https://a.com/b/c/d.extasd')).toBeFalsy(); + expect(urlHasFileExtension('https://a.com/b/c/d32.ext1asd')).toBeFalsy(); + expect(urlHasFileExtension('https://a.com/b/c/d12.ext1-asd')).toBeFalsy(); + expect(urlHasFileExtension('https://a.com/b/c/d12.ER')).toBeFalsy(); + }); +}); + describe('divideCoordinates', () => { it('return coordinates divided by a number', () => { const geojson = {