diff --git a/campaigns-schema.yaml b/campaigns-schema.yaml index bf4a4b9..f9f3079 100644 --- a/campaigns-schema.yaml +++ b/campaigns-schema.yaml @@ -8,6 +8,7 @@ deployments: name: required: true type: string + description: Deployment identifier platforms: required: true type: list @@ -17,21 +18,33 @@ deployments: name: required: true type: string + description: Platform name tsv_format: required: false type: boolean + description: Whether the files are in TSV format rename_as_ict: required: false - type: boolean + description: Whether the files are ICARTT, but have another file extension. If true, they will be renamed to .ict after the download. process_as_ict: required: false type: boolean + description: Whether a .txt file should be processed as a .ict file. coords_divisor: required: false type: integer + description: It will divide the coordinates values by an integer number. data_start_line_fix: required: false type: integer + description: Some ICARTT files have a wrong data start line. This field value will be summed to the ICARTT start line to fix the error. + header_content: + required: false + type: string + description: Set a new value to the ICARTT header, as some files don't have a header. files: required: true - type: list \ No newline at end of file + type: list + items: + type: string + description: URL of the file \ No newline at end of file diff --git a/campaigns/SOLVE/deployments.yaml b/campaigns/SOLVE/deployments.yaml new file mode 100644 index 0000000..f4a754a --- /dev/null +++ b/campaigns/SOLVE/deployments.yaml @@ -0,0 +1,56 @@ +--- +deployments: +- name: SOLVE-D1_1999 + platforms: + - name: DC-8 + coords_divisor: 1000 + tsv_format: true + rename_as_ict: true + files: + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/11/SOLVE-DG19991116.DC8 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/11/SOLVE-DG19991119.DC8 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/11/SOLVE-DG19991122.DC8 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/11/SOLVE-DG19991130.DC8 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/12/SOLVE-DG19991202.DC8 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/12/SOLVE-DG19991205.DC8 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/12/SOLVE-DG19991207.DC8 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/12/SOLVE-DG19991210.DC8 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/12/SOLVE-DG19991212.DC8 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/12/SOLVE-DG19991214.DC8 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_DC8_Data_1/1999/12/SOLVE-DG19991216.DC8 + - name: ER-2 + coords_divisor: 1000 + tsv_format: true + rename_as_ict: true + files: + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/1999/12/SOLVE-FP19991211.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/1999/12/SOLVE-FP19991215.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/1999/12/SOLVE-FP19991216.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000106.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000109.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000111.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000114.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000120.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000123.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000127.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/01/SOLVE-FP20000131.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/02/SOLVE-FP20000202.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/02/SOLVE-FP20000203.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/02/SOLVE-FP20000226.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/03/SOLVE-FP20000305.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/03/SOLVE-FP20000307.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/03/SOLVE-FP20000311.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/03/SOLVE-FP20000312.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/03/SOLVE-FP20000316.ER2 + - https://asdc.larc.nasa.gov/data/SOLVE/MetNav_AircraftInSitu_ER2_Data_1/2000/03/SOLVE-FP20000318.ER2 +- name: SOLVE-D2_2002 + platforms: + - name: DC-8 + coords_divisor: 10000 + tsv_format: true + rename_as_ict: true + header_content: a,b,latitude,longitude,speed + files: + - https://asdc.larc.nasa.gov/data/SOLVE_II/MetNav_AircraftInSitu_DC8_Data_1/2002/12/SOLVE-ID20021213__FLT.DC8 + - https://asdc.larc.nasa.gov/data/SOLVE_II/MetNav_AircraftInSitu_DC8_Data_1/2002/12/SOLVE-ID20021217__FLT.DC8 + - https://asdc.larc.nasa.gov/data/SOLVE_II/MetNav_AircraftInSitu_DC8_Data_1/2002/12/SOLVE-ID20021219__FLT.DC8 \ No newline at end of file diff --git a/campaigns/SONEX/deployments.yaml b/campaigns/SONEX/deployments.yaml new file mode 100644 index 0000000..b401f31 --- /dev/null +++ b/campaigns/SONEX/deployments.yaml @@ -0,0 +1,25 @@ +--- +deployments: +- name: SONEX-D1_1997 + platforms: + - name: DC-8 + coords_divisor: 1000 + tsv_format: true + rename_as_ict: true + files: + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971007.DC8 + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971009.DC8 + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971013.DC8 + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971015.DC8 + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971018.DC8 + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971020.DC8 + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971023.DC8 + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971025.DC8 + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971028.DC8 + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971029.DC8 + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/10/SONEX-FP19971031.DC8 + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/11/SONEX-FP19971103.DC8 + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/11/SONEX-FP19971105.DC8 + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/11/SONEX-FP19971109.DC8 + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/11/SONEX-FP19971110.DC8 + - https://asdc.larc.nasa.gov/data/SONEX/MetNav_AircraftInSitu_DC8_Data_1/1997/11/SONEX-FP19971112.DC8 \ No newline at end of file diff --git a/task/makeCSV.js b/task/makeCSV.js index 7f5fe82..94f8d7d 100755 --- a/task/makeCSV.js +++ b/task/makeCSV.js @@ -18,13 +18,23 @@ const makeCSV = (platformPath) => { // process .ict files const ictFiles = findFiles(platformPath, '.ict'); ictFiles.forEach( - (f) => splitICTFile(f, platformConfig.tsv_format, platformConfig.data_start_line_fix) + (f) => splitICTFile( + f, + platformConfig.tsv_format, + platformConfig.data_start_line_fix, + platformConfig.header_content + ) ); // some platforms have txt files formatted as icartt if (platformConfig.process_as_ict) { txtFiles.forEach( - (f) => splitICTFile(f, platformConfig.tsv_format, platformConfig.data_start_line_fix) + (f) => splitICTFile( + f, + platformConfig.tsv_format, + platformConfig.data_start_line_fix, + platformConfig.header_content + ) ); } }; diff --git a/task/src/process.js b/task/src/process.js index 05510fc..8cfee1d 100644 --- a/task/src/process.js +++ b/task/src/process.js @@ -31,17 +31,21 @@ const getPropertiesFromPath = (dir) => { * @param {Number} dataStartLineFix - sum or substract a value to/from the dataStartLine * value in the ict header */ -const getDataContentFromICT = (filename, dataStartLineFix = 0) => { +const getDataContentFromICT = (filename, dataStartLineFix = 0, replaceHeaderContent = '') => { const file = fs.readFileSync(filename); let content = file.toString().split('\n'); let dataStartLine; + let header; // some campaigns have a different header identifier if (content[0] === '/begin_header') { dataStartLine = content.indexOf('/end_header'); - const header = content[dataStartLine - 2].replace('/fields=', ''); + header = content[dataStartLine - 2].replace('/fields=', ''); return [header, ...content.slice(dataStartLine + 1)].join('\n'); } dataStartLine = Number(content[0].split(/\s*,?\s*1001/g)[0]); + if (replaceHeaderContent) { + return [replaceHeaderContent, ...content.slice(dataStartLine - 1)].join('\n'); + } content = content.slice(dataStartLine - 1 + dataStartLineFix).join('\n'); return content; }; @@ -53,8 +57,13 @@ const getDataContentFromICT = (filename, dataStartLineFix = 0) => { * @param {Number} dataStartLineFix - sum or substract a value to/from the dataStartLine * value in the ict header */ -const splitICTFile = (filename, isTSVFormatted = false, dataStartLineFix = 0) => { - let content = getDataContentFromICT(filename, dataStartLineFix); +const splitICTFile = ( + filename, + isTSVFormatted = false, + dataStartLineFix = 0, + replaceHeaderContent = '' +) => { + let content = getDataContentFromICT(filename, dataStartLineFix, replaceHeaderContent); if (isTSVFormatted) { content = tsv2csv(content); diff --git a/task/tests/validate-yaml.test.js b/task/tests/validate-yaml.test.js index caa2302..bf2107a 100644 --- a/task/tests/validate-yaml.test.js +++ b/task/tests/validate-yaml.test.js @@ -22,6 +22,7 @@ const structure = array( process_as_ict: optional(boolean()), coords_divisor: optional(number()), data_start_line_fix: optional(number()), + header_content: optional(string()), files: array(string()), }) ),