From 1896d64930c8f2dcdffbafce5437d5e198f0c7f5 Mon Sep 17 00:00:00 2001 From: Wille Marcel Date: Mon, 20 May 2024 11:36:35 -0300 Subject: [PATCH] Add ARCTAS campaign and change code to download and process it --- campaigns/ARCTAS/deployments.yaml | 71 +++++++++++++++++++++++++++++++ campaigns/ARCTAS/static.csv | 2 +- task/package.json | 1 + task/src/download.js | 9 +++- task/src/process.js | 11 ++++- task/yarn.lock | 5 +++ 6 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 campaigns/ARCTAS/deployments.yaml diff --git a/campaigns/ARCTAS/deployments.yaml b/campaigns/ARCTAS/deployments.yaml new file mode 100644 index 0000000..dd52fdd --- /dev/null +++ b/campaigns/ARCTAS/deployments.yaml @@ -0,0 +1,71 @@ +--- +deployments: +- name: ARCTAS-D1_2008-Apr + platforms: + - name: DC-8 + tsv_format: true + files: + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080401_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080404_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080405_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080408_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080409_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080412_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080416_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080417_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080419_R1.zip + - name: P-3 + tsv_format: true + files: + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080331_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080401_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080406_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080408_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080409_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080413_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080415_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080419_R2.zip +- name: ARCTAS-D2_2008-Jun_California + platforms: + - name: DC-8 + tsv_format: true + files: + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080624_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080622_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080620_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080618_R1.zip + - name: P-3 + tsv_format: true + files: + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080622_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080624_R2.zip +- name: ARCTAS-D3_2008-Jun + platforms: + - name: DC-8 + tsv_format: true + files: + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080626_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080629_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080701_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080704_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080705_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080708_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080709_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080710_R1.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_DC8_Data_1/ARCTAS-NAV_DC8_20080713_R1.zip + - name: P-3 + tsv_format: true + files: + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080626_R2_FLT13.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080626_R2_FLT14.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080628_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080629_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080630_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080702_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080703_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080706_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080707_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080709_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080710_R2.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080712_R2_FLT24.zip + - https://asdc.larc.nasa.gov/data/ARCTAS/MetNav_AircraftInSitu_P3B_Data_1/ARCTAS-PDS_P3B_20080712_R2_FLT25.zip \ No newline at end of file diff --git a/campaigns/ARCTAS/static.csv b/campaigns/ARCTAS/static.csv index 60e3561..790d5e0 100644 --- a/campaigns/ARCTAS/static.csv +++ b/campaigns/ARCTAS/static.csv @@ -11,7 +11,7 @@ ARCTAS,ARCTAS-D1_2008-Apr,Balloon Launch Site,44.95,-59.92 ARCTAS,ARCTAS-D1_2008-Apr,Balloon Launch Site,53.55,-114.11 ARCTAS,ARCTAS-D1_2008-Apr,Balloon Launch Site,60.7,-135.07 ARCTAS,ARCTAS-D1_2008-Apr,Balloon Launch Site,41.4,-71.5 -ARCTAS,ARCTAS-D1_2008-Apr,Balloon Launch Site,71.32,156.6 +ARCTAS,ARCTAS-D1_2008-Apr,Balloon Launch Site,71.32,-156.6 ARCTAS,ARCTAS-D2_2008-Jun,Field Site,71.3,-156.74 ARCTAS,ARCTAS-D2_2008-Jun,Balloon Launch Site,41.4,-71.5 ARCTAS,ARCTAS-D2_2008-Jun,Balloon Launch Site,39.95,-105.25 diff --git a/task/package.json b/task/package.json index 83743aa..566d0d6 100644 --- a/task/package.json +++ b/task/package.json @@ -13,6 +13,7 @@ "dependencies": { "@mapbox/geojson-merge": "^1.1.1", "@turf/distance": "^6.5.0", + "adm-zip": "^0.5.12", "csv2geojson": "^5.1.2", "d3-dsv": "1.0.1", "download": "8.0.0", diff --git a/task/src/download.js b/task/src/download.js index 2a2c9e8..2c74e8d 100644 --- a/task/src/download.js +++ b/task/src/download.js @@ -1,6 +1,6 @@ const fs = require('fs'); const path = require('path'); - +const AdmZip = require('adm-zip'); const download = require('download'); const { getPlatformConfig, readCampaignYaml } = require('./utils'); @@ -12,6 +12,13 @@ const downloadFile = async (url, dir) => { dir, { headers: { Authorization: `Bearer ${process.env.EARTH_DATA_TOKEN}` } } ); + // if the file is a zip, decompress it + if (url.endsWith('.zip')) { + const filePath = path.join(dir, path.basename(url)); + const zip = new AdmZip(filePath); + zip.extractAllTo(dir); + fs.unlinkSync(filePath); + } }; const downloadCampaign = (campaignPath) => { diff --git a/task/src/process.js b/task/src/process.js index a14fc7b..173d684 100644 --- a/task/src/process.js +++ b/task/src/process.js @@ -33,12 +33,12 @@ const splitICTFile = (filename, isTSVFormatted = false) => { if (isTSVFormatted) { content = tsv2csv(content); } - let columnNotFound = true; // ICART files can have different column names for the start time const possibleFirstColumnNames = [ - 'Time_Start,', 'Time_mid,', 'Start_UTC,', 'TIME_NP,', 'UTC,', 'Time', 'UT', + 'Time_Start,', 'Time_mid,', 'Time', 'UTC,', 'Start_UTC,', 'TIME_NP,', 'UT', ]; + let columnNotFound = true; possibleFirstColumnNames.forEach((col) => { if (content.indexOf(col) !== -1 && columnNotFound) { content = content.substr(content.lastIndexOf(col)); @@ -53,6 +53,10 @@ const splitICTFile = (filename, isTSVFormatted = false) => { .replace(',Lon,', ',longitude,') .replace(', LAT,', ',latitude,') .replace(', LONG,', ',longitude,') + .replace(',LAT,', ',latitude,') + .replace(',LON,', ',longitude,') + .replace(',FMS_LAT,', ',latitude,') + .replace(',FMS_LON,', ',longitude,') .replace(',GGLAT,', ',latitude,') .replace(',GGLON,', ',longitude,') .replace(',GLAT,', ',latitude,') @@ -68,6 +72,9 @@ const splitICTFile = (filename, isTSVFormatted = false) => { .replace(', Latitude,', ',latitude,') .replace(', Longitude,', ',longitude,'); + const header = content.substr(0, content.indexOf('\n')).replaceAll(',,', ','); + content = `${header}${content.substr(content.indexOf('\n'))}`; + const newFileName = filename.endsWith('.ict') ? filename.replace('.ict', '.csv') : `${filename}.csv`; diff --git a/task/yarn.lock b/task/yarn.lock index 2eea81e..a51a5bc 100644 --- a/task/yarn.lock +++ b/task/yarn.lock @@ -820,6 +820,11 @@ adler-32@~1.3.0: resolved "https://registry.yarnpkg.com/adler-32/-/adler-32-1.3.1.tgz#1dbf0b36dda0012189a32b3679061932df1821e2" integrity sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A== +adm-zip@^0.5.12: + version "0.5.12" + resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.12.tgz#87786328e91d54b37358d8a50f954c4cd73ba60b" + integrity sha512-6TVU49mK6KZb4qG6xWaaM4C7sA/sgUMLy/JYMOzkcp3BvVLpW0fXDFQiIzAuxFCt/2+xD7fNIiPFAoLZPhVNLQ== + ajv@^6.12.4: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4"