From 6164745674f3bb4258081d4f97d2a25005c38665 Mon Sep 17 00:00:00 2001 From: Wille Marcel Date: Fri, 16 Aug 2024 19:01:21 -0300 Subject: [PATCH] Add new campaings and improve ICT split method --- campaigns/AAOE/deployments.yaml | 46 ++++++++++++++ campaigns/AASE/deployments.yaml | 109 ++++++++++++++++++++++++++++++++ campaigns/ATom/deployments.yaml | 65 +++++++++++++++++++ task/src/process.js | 28 ++------ 4 files changed, 224 insertions(+), 24 deletions(-) create mode 100644 campaigns/AAOE/deployments.yaml create mode 100644 campaigns/AASE/deployments.yaml create mode 100644 campaigns/ATom/deployments.yaml diff --git a/campaigns/AAOE/deployments.yaml b/campaigns/AAOE/deployments.yaml new file mode 100644 index 0000000..ec515a5 --- /dev/null +++ b/campaigns/AAOE/deployments.yaml @@ -0,0 +1,46 @@ +--- +deployments: +- name: AAOE-D1_1987 + platforms: + - name: ER-2 + tsv_format: true + rename_as_ict: true + coords_divisor: 1000 + files: + - https://espoarchive.nasa.gov/archive/download/FP19870812.ER2 + - https://espoarchive.nasa.gov/archive/download/FP19870814.ER2 + - https://espoarchive.nasa.gov/archive/download/FP19870815.ER2 + - https://espoarchive.nasa.gov/archive/download/FP19870817.ER2 + - https://espoarchive.nasa.gov/archive/download/FP19870828.ER2 + - https://espoarchive.nasa.gov/archive/download/FP19870830.ER2 + - https://espoarchive.nasa.gov/archive/download/FP19870902.ER2 + - https://espoarchive.nasa.gov/archive/download/FP19870904.ER2 + - https://espoarchive.nasa.gov/archive/download/FP19870909.ER2 + - https://espoarchive.nasa.gov/archive/download/FP19870916.ER2 + - https://espoarchive.nasa.gov/archive/download/FP19870920.ER2 + - https://espoarchive.nasa.gov/archive/download/FP19870921.ER2 + - https://espoarchive.nasa.gov/archive/download/FP19870922.ER2 + - https://espoarchive.nasa.gov/archive/download/FP19870929.ER2 + - https://espoarchive.nasa.gov/archive/download/FP19871003.ER2 + - name: DC-8 + tsv_format: true + rename_as_ict: true + coords_divisor: 1000 + files: + - https://espoarchive.nasa.gov/archive/download/NP19870819.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19870822.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19870828.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19870830.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19870902.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19870905.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19870908.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19870911.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19870914.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19870916.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19870919.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19870921.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19870924.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19870926.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19870929.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19871002.DC8 + - https://espoarchive.nasa.gov/archive/download/NP19871004.DC8 \ No newline at end of file diff --git a/campaigns/AASE/deployments.yaml b/campaigns/AASE/deployments.yaml new file mode 100644 index 0000000..33cfa06 --- /dev/null +++ b/campaigns/AASE/deployments.yaml @@ -0,0 +1,109 @@ +--- +deployments: +- name: AASE-D1_1988-Dec + platforms: + - name: ER-2 + tsv_format: true + rename_as_ict: true + coords_divisor: 1000 + files: + - https://espo.nasa.gov/aase/archive/download/FP19881229.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19881231.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890103.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890106.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890107.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890112.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890116.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890119.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890120.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890124.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890125.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890130.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890207.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890208.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890209.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890210.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890220.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19890221.ER2 + - name: DC-8 + tsv_format: true + rename_as_ict: true + coords_divisor: 1000 + files: + - https://espo.nasa.gov/aase/archive/download/NP19890102.DC8 + - https://espo.nasa.gov/aase/archive/download/NP19890106.DC8 + - https://espo.nasa.gov/aase/archive/download/NP19890109.DC8 + - https://espo.nasa.gov/aase/archive/download/NP19890111.DC8 + - https://espo.nasa.gov/aase/archive/download/NP19890114.DC8 + - https://espo.nasa.gov/aase/archive/download/NP19890117.DC8 + - https://espo.nasa.gov/aase/archive/download/NP19890119.DC8 + - https://espo.nasa.gov/aase/archive/download/NP19890124.DC8 + - https://espo.nasa.gov/aase/archive/download/NP19890126.DC8 + - https://espo.nasa.gov/aase/archive/download/NP19890129.DC8 + - https://espo.nasa.gov/aase/archive/download/NP19890131.DC8 + - https://espo.nasa.gov/aase/archive/download/NP19890202.DC8 + - https://espo.nasa.gov/aase/archive/download/NP19890205.DC8 + - https://espo.nasa.gov/aase/archive/download/NP19890207.DC8 + - https://espo.nasa.gov/aase/archive/download/NP19890209.DC8 + - https://espo.nasa.gov/aase/archive/download/NP19890215.DC8 +- name: AASE-D2_1991-Sep + platforms: + - name: ER-2 + tsv_format: true + rename_as_ict: true + coords_divisor: 1000 + files: + - https://espo.nasa.gov/aase/archive/download/FP19910917.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19910919.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19911004.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19911006.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19911008.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19911010.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19911012.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19911014.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19911102.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19911104.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19911208.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19911210.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19911212.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19911214.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920104.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920106.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920108.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920111.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920112.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920116.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920120.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920213.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920215.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920217.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920222.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920315.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920318.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920320.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920322.ER2 + - https://espo.nasa.gov/aase/archive/download/FP19920326.ER2 + - name: DC-8 + tsv_format: true + rename_as_ict: true + coords_divisor: 1000 + files: + - https://espo.nasa.gov/aase/archive/download/DF19920108.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920111.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920114.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920116.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920119.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920122.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920123.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920128.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920129.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920212.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920214.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920217.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920220.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920222.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920310.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920312.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920314.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920318.DC8 + - https://espo.nasa.gov/aase/archive/download/DF19920320.DC8 \ No newline at end of file diff --git a/campaigns/ATom/deployments.yaml b/campaigns/ATom/deployments.yaml new file mode 100644 index 0000000..15d524b --- /dev/null +++ b/campaigns/ATom/deployments.yaml @@ -0,0 +1,65 @@ +--- +deployments: +- name: ATom-D1_2016 + platforms: + - name: DC-8 + coords_divisor: 10000 + files: + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160823_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160822_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160820_R2.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160817_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160815_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160812_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160808_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160806_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160803_R3.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160801_R2.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20160729_R1.ict +- name: ATom-D2_2017-Jan + platforms: + - name: DC-8 + coords_divisor: 10000 + files: + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170221_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170219_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170218_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170215_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170213_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170210_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170205_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170203_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170201_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170129_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170126_R1.ict +- name: ATom-D3_2017-Sep + platforms: + - name: DC-8 + coords_divisor: 10000 + files: + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20171017_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20171014_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20171011_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20171008_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20171006_R2.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20171004_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20171001_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20170928_R1.ict +- name: ATom-D4_2018 + platforms: + - name: DC-8 + coords_divisor: 100000 + files: + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180521_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180519_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180518_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180517_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180514_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180512_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180509_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180506_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180503_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180501_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180429_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180427_R1.ict + - https://data.ornldaac.earthdata.nasa.gov/protected/atom/ATom_MMS_Instrument_Data/data/MMS-1HZ_DC8_20180424_R1.ict diff --git a/task/src/process.js b/task/src/process.js index 757e5b2..ca25a4c 100644 --- a/task/src/process.js +++ b/task/src/process.js @@ -30,34 +30,13 @@ const getPropertiesFromPath = (dir) => { */ const splitICTFile = (filename, isTSVFormatted = false) => { const file = fs.readFileSync(filename); - let content = file.toString(); + let content = file.toString().split('\n'); + const dataStartLine = Number(content[0].split(/\s*,?\s*1001/g)[0]); + content = content.slice(dataStartLine - 1).join('\n'); if (isTSVFormatted) { content = tsv2csv(content); } - // ICART files can have different column names for the start time - const possibleFirstColumnNames = [ - 'index, time_start,', - 'Gps_time,', - 'Time_Start,', - 'TIME_START,', - 'StartTime_UTsec,', - 'Time_mid,', - 'Gps_time_midpoint,', - 'Start_UTC,', - 'UTC,', - 'Time', - 'TIME_NP,', - 'UT', - ]; - let columnNotFound = true; - possibleFirstColumnNames.forEach((col) => { - if (content.indexOf(col) !== -1 && columnNotFound) { - content = content.substr(content.lastIndexOf(col)); - columnNotFound = false; - } - }); - // some files have different column names for latitude and longitude content = content .replace(',Lat,', ',latitude,') @@ -65,6 +44,7 @@ const splitICTFile = (filename, isTSVFormatted = false) => { .replace(',Lon,', ',longitude,') .replace(', LAT,', ',latitude,') .replace(', LONG,', ',longitude,') + .replace(',LONG,', ',longitude,') .replace(',LAT,', ',latitude,') .replace(',LON,', ',longitude,') .replace(',FMS_LAT,', ',latitude,')